Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: stop flushing the CUDA allocator on every prompt encode under block offload (#14777)
Daniel Han9f5010c423c3
diff --git a/studio/backend/core/inference/diffusion_block_graph.py b/studio/backend/core/inference/diffusion_block_graph.pyindex 80302c9bd08..02e006e59fd 100644--- a/studio/backend/core/inference/diffusion_block_graph.py+++ b/studio/backend/core/inference/diffusion_block_graph.py@@ -249,7 +249,7 @@ _UNSEEN = object()class _Entry:- __slots__ = ("graph", "static_in", "static_out", "out_spec", "replays", "slots")+ __slots__ = ("graph", "static_in", "static_out", "out_spec", "replays", "slots", "pool")import weakref@@ -607,8 +607,9 @@ class BlockGraph:graph = torch.cuda.CUDAGraph()pool = self.shared.graph_pool()with torch.cuda.stream(stream):- before = torch.cuda.memory_reserved()with _capturing():+ # read after the entry, which first hands back any retired graph pool+ before = torch.cuda.memory_reserved()try:# Inside the try: a capture_begin that raised after taking the pool is abandoned too.graph.capture_begin(pool = pool, capture_error_mode = "thread_local")@@ -642,6 +643,7 @@ class BlockGraph:entry.slots = (slot, out_slot)self.shared.hold(entry.slots)entry.graph = graph+ entry.pool = poolentry.out_spec = out_specentry.replays = 0if any(k[0] == full[0] for k in self.cache):@@ -656,8 +658,7 @@ class BlockGraph:note_capture(self._unsloth_capture_device)_LIVE_BLOCKS.add(self)while len(self.cache) > self.max_graphs:- _, evicted = self.cache.popitem(last = False)- self._forget(evicted)+ _drop_entries(self, [self.cache.popitem(last = False)[1]], evicting = True)self.stats["evictions"] += 1return entry@@ -666,9 +667,9 @@ class BlockGraph:self.shared.drop(getattr(entry, "slots", ()))def _forget_all(self) -> None:- for entry in self.cache.values():- self._forget(entry)+ entries = list(self.cache.values())self.cache.clear()+ _drop_entries(self, entries)def _replay(self, entry: _Entry, live: list) -> Any:for dst, src in zip(entry.static_in, live):@@ -722,7 +723,7 @@ class _SegRecord:class _SegEntry:- __slots__ = ("graph", "static_in", "outputs", "record", "slots", "replays")+ __slots__ = ("graph", "static_in", "outputs", "record", "slots", "replays", "pool")def _root_key_extra(root: Any) -> Any:@@ -815,9 +816,9 @@ class SegmentGraph:self.versions: list = []def reset(self) -> None:- for entry in self.cache.values():- self._forget(entry)+ entries = list(self.cache.values())self.cache.clear()+ _drop_entries(self, entries)self.seen.clear()self.refused.clear()self._reset_forward()@@ -1049,8 +1050,9 @@ class SegmentGraph:graph = torch.cuda.CUDAGraph()pool = self.shared.graph_pool()with torch.cuda.stream(stream):- before = torch.cuda.memory_reserved()with _capturing():+ # read after the entry, which first hands back any retired graph pool+ before = torch.cuda.memory_reserved()try:graph.capture_begin(pool = pool, capture_error_mode = "thread_local")try:@@ -1081,6 +1083,7 @@ class SegmentGraph:)return Noneentry.graph = graph+ entry.pool = poolentry.outputs = outputsentry.record = recentry.slots = (slot,)@@ -1093,8 +1096,7 @@ class SegmentGraph:self._unsloth_capture_device = capture_device()note_capture(self._unsloth_capture_device)while len(self.cache) > self.max_graphs:- _, evicted = self.cache.popitem(last = False)- self._forget(evicted)+ _drop_entries(self, [self.cache.popitem(last = False)[1]], evicting = True)return entry@@ -1413,14 +1415,37 @@ class BlockGraphSet:return int(sum(st.pool_bytes + st.static_bytes for st in self._all_shared()) + slots)-def _release_cached() -> None:+def _drop_entries(+ owner: Any,+ entries: list,+ evicting: bool = False,+) -> None:+ """The one way a block or segment graph drops recordings (eviction, poison, reset): forget them, then count their+ pool as retired, so no capture records beside its cached segments before a flush. An eviction from the current+ pool is not counted (the entry just recorded into it keeps it alive); any other drop is."""+ if not entries:+ return+ for entry in entries:+ owner._forget(entry)+ current = getattr(getattr(owner, "shared", None), "pool", None)+ if (+ evicting+ and current is not None+ and all(getattr(e, "pool", None) == current for e in entries)+ ):+ returntry:- import gc+ from .diffusion_cuda_graph import note_pool_retired+ note_pool_retired()+ except Exception: # noqa: BLE001+ pass- gc.collect()- torch = _torch()- if torch.cuda.is_available():- torch.cuda.empty_cache()++def _release_cached() -> None:+ try:+ from .diffusion_cuda_graph import release_cached_outside_capture+ if _torch().cuda.is_available():+ release_cached_outside_capture()except Exception: # noqa: BLE001passdiff --git a/studio/backend/core/inference/diffusion_cuda_graph.py b/studio/backend/core/inference/diffusion_cuda_graph.pyindex 1a319ba973c..8dc6fed8d92 100644--- a/studio/backend/core/inference/diffusion_cuda_graph.py+++ b/studio/backend/core/inference/diffusion_cuda_graph.py@@ -217,12 +217,29 @@ def _capturing():The depth MUST be raised before ``torch.cuda.graph`` is entered, which is what makes theordering above hold; keep this the outermost of the two context managers. Raises``CaptureRefused`` (depth untouched, nothing recorded) when replica-side work did not drain in time.++ Every capture (whole step, planned or offloaded re-record, per block, per segment) enters here, so this is also+ where a retired graph pool is handed back before anything records (``_settle_retired_pools``)."""global _CAPTURE_DEPTHown = getattr(_OUTSIDE_LOCAL, "depth", 0)+ seen = _retired_pools_seen()with _CAPTURE_IDLE:- _wait_outside_drained(own)- _CAPTURE_DEPTH += 1+ for _ in range(_ENTRY_SETTLE_TRIES):+ if seen is not None:+ _settle_retired_pools(seen)+ _wait_outside_drained(own)+ # Retirement takes _RETIRE_LOCK, so one is either counted before this check (flush again, still under+ # the capture lock) or lands once the depth is up (due for the next capture): never in between.+ with _RETIRE_LOCK:+ if _RETIRED_POOLS[1] >= _RETIRED_POOLS[0]:+ _CAPTURE_DEPTH += 1+ break+ seen = _RETIRED_POOLS[0]+ else:+ raise CaptureRefused(+ "graph pools kept retiring while this capture entered; capture deferred"+ )try:yieldfinally:@@ -293,6 +310,149 @@ def hold_off_capture():_CAPTURE_LOCK.release()+# A retired pool's segments stay cached until empty_cache, which cannot run while a capture records, so a capture+# beside them can OOM. Rule: no capture records until every pool retired before its entry was flushed. Counted, not+# flagged, so a retirement landing mid-flush stays due.+_RETIRE_LOCK = threading.Lock()+_RETIRED_POOLS = [0, 0] # [retirements counted, retirements flushed]+_POOL_FLUSHES = [0] # retired-pool flushes performed (diagnostics, tests)+# A deferred release owes a cuBLAS workspace clear; the next entry flush does it.+_CUBLAS_CLEAR_DUE = [False]+# Wait (not eager fallback) for another thread's recording to end: eager CUDA work mid-capture can invalidate it.+RETIRED_FLUSH_WAIT_S = OUTSIDE_WAIT_S+# Entry flush attempts before the capture is refused.+_ENTRY_SETTLE_TRIES = 8+++def note_pool_retired() -> None:+ """Count a retired graph pool. Call once the graphs that held it are dropped, never before."""+ with _RETIRE_LOCK:+ _RETIRED_POOLS[0] += 1+++def retired_pool_due() -> bool:+ with _RETIRE_LOCK:+ return _RETIRED_POOLS[0] != _RETIRED_POOLS[1]+++def _retired_pools_seen() -> Optional[int]:+ """The retirement count a flush starting now covers, None when nothing is due. Read BEFORE ``gc.collect``, so+ every graph counted is already unreferenced (or in a cycle that collect frees) when the flush runs."""+ with _RETIRE_LOCK:+ seen = _RETIRED_POOLS[0]+ if seen == _RETIRED_POOLS[1]:+ return None+ gc.collect() # outside _CAPTURE_LOCK: a finalizer may take it+ return seen+++def _flush_retired_locked(seen: int) -> bool:+ """Under ``_CAPTURE_LOCK`` with no capture recording: synchronize, clear the cuBLAS workspaces a deferred release+ owed, flush, acknowledge up to ``seen``."""+ with _RETIRE_LOCK:+ clear_cublas, _CUBLAS_CLEAR_DUE[0] = _CUBLAS_CLEAR_DUE[0], False+ try:+ torch = _torch()+ cuda = getattr(torch, "cuda", None)+ if cuda is not None and cuda.is_available():+ sync = getattr(cuda, "synchronize", None)+ if callable(sync):+ sync()+ if clear_cublas:+ clear = getattr(getattr(torch, "_C", None), "_cuda_clearCublasWorkspaces", None)+ if callable(clear):+ clear()+ empty = getattr(cuda, "empty_cache", None)+ if callable(empty):+ empty()+ except Exception: # noqa: BLE001 - not flushed: stays due, the cuBLAS clear with it+ if clear_cublas:+ with _RETIRE_LOCK:+ _CUBLAS_CLEAR_DUE[0] = True+ return False+ with _RETIRE_LOCK:+ _RETIRED_POOLS[1] = max(_RETIRED_POOLS[1], seen)+ _POOL_FLUSHES[0] += 1+ return True+++def _settle_retired_pools(seen: int) -> None:+ """Under ``_CAPTURE_IDLE``, from ``_capturing``: flush the pools retired up to ``seen`` before this capture records.+ Waits (lock released) for another thread's recording to end, since that one prohibits the flush; raises+ ``CaptureRefused`` (depth untouched, nothing recorded) when it does not end in time or the flush failed."""+ with _RETIRE_LOCK:+ if _RETIRED_POOLS[1] >= seen:+ return # another capture flushed them already: no wait on its recording+ if not _CAPTURE_IDLE.wait_for(lambda: _CAPTURE_DEPTH == 0, timeout = RETIRED_FLUSH_WAIT_S):+ raise CaptureRefused(+ "a retired graph pool could not be flushed (another capture is recording); capture deferred"+ )+ with _RETIRE_LOCK:+ if _RETIRED_POOLS[1] >= seen:+ return # another capture flushed them while this one waited+ if not _flush_retired_locked(seen):+ raise CaptureRefused("a retired graph pool could not be flushed; capture deferred")+++def flush_retired_pools() -> bool:+ """Best effort, outside any capture: hand retired pools back now rather than at the next capture's entry. False+ (still due) while another thread records."""+ seen = _retired_pools_seen()+ if seen is None:+ return True+ with hold_off_capture() as safe:+ if not safe:+ return False+ return _flush_retired_locked(seen)+++def flush_now_locked(clear_cublas: bool = False) -> bool:+ """For a caller already holding ``hold_off_capture()`` (True): flush now, and acknowledge every retirement counted+ before it, so the next capture's entry does not flush again."""+ with _RETIRE_LOCK:+ seen = _RETIRED_POOLS[0]+ if clear_cublas:+ _CUBLAS_CLEAR_DUE[0] = True+ gc.collect()+ return _flush_retired_locked(seen)+++def release_cached_outside_capture(clear_cublas: bool = False) -> bool:+ """The one way graph memory goes back outside a capture's entry: ``empty_cache`` (and the cuBLAS workspaces) now,+ under the capture lock with no capture recording, acknowledging the retirements counted before it; or, while+ another thread records (either call is prohibited inside its recording and can invalidate it), counted as a+ retired pool with the cuBLAS clear owed, so the next capture's entry flush does both."""+ # Owed first, in one step with the cuBLAS debt: a capture entering from here on sees it due, whether or not this+ # flush gets the lock; a successful flush below acknowledges it.+ with _RETIRE_LOCK:+ _RETIRED_POOLS[0] += 1+ seen = _RETIRED_POOLS[0]+ if clear_cublas:+ _CUBLAS_CLEAR_DUE[0] = True+ gc.collect() # outside _CAPTURE_LOCK: a finalizer may take it+ with hold_off_capture() as safe:+ if safe and _flush_retired_locked(seen):+ return True+ return False+++def _note_dropped_pools(tokens: list) -> None:+ """Graph entries just left a step cache: count a retirement when one of their pools no live graph records into+ (a pool still shared with a live graph keeps its freed blocks for that pool's next capture)."""+ try:+ if not tokens:+ return+ live = [+ getattr(entry, "pool_token", None)+ for wrapper in tuple(_LIVE_WRAPPERS)+ for entry in tuple(wrapper.cache.values())+ ]+ if any(token is None or token not in live for token in tokens):+ note_pool_retired()+ except Exception: # noqa: BLE001 - unsure: count it, the cost is one flush+ note_pool_retired()++# Raised while a whole step warms up or records: per-block graphs then run their compute, not nested replays._STEP_RECORDING = [0]@@ -545,6 +705,8 @@ def retire_failed_capture((_COLLIDED_GRAPHS if begun_elsewhere else _FAILED_GRAPHS).append(graph)if pool is not None and _POOL_BOX[0] == pool:_POOL_BOX[0] = None+ if not begun_elsewhere:+ note_pool_retired() # whatever the failed recording took stays cached until the next capture's entry flushdef _nvfp4_flashinfer_linears(module: Any) -> list:@@ -985,23 +1147,15 @@ class GraphedForward:def _release(self) -> None:"""Return the dropped graphs' pool segments to the device; ``gc.collect`` alone leaves them in the private pool."""- try:- gc.collect()- torch = _torch()- cuda = getattr(torch, "cuda", None)- if getattr(self, "placement", None) is not None:- # the recordings ran on their own capture stream, whose cuBLAS workspace outlives them- clear = getattr(torch._C, "_cuda_clearCublasWorkspaces", None)- if callable(clear):- clear()- if cuda is not None and hasattr(cuda, "empty_cache"):- cuda.empty_cache()- except Exception: # noqa: BLE001 - best effort- pass+ # the recordings ran on their own capture stream, whose cuBLAS workspace outlives them+ release_cached_outside_capture(clear_cublas = getattr(self, "placement", None) is not None)def reset(self) -> "GraphedForward":"""Drop every captured graph (weights changed under us: LoRA load, unload, adapter switch)."""+ dropped = bool(self.cache)self.cache.clear()+ if dropped:+ note_pool_retired() # counted where it drops; the _release below flushes and acknowledges itself._recorded.clear()# Nothing recorded is left to replay: every scope's first render starts over (captures counted afresh).self._sealed_scopes.clear()@@ -1027,7 +1181,9 @@ class GraphedForward:# entry is now unreachable and only pins its statics, outputs and slice of the pool for the# life of the load. Freeing is left to the caller's ``_release``, which runs once the failed# capture's own frames are gone, so both go back in one pass.+ dropped = [getattr(e, "pool_token", None) for e in self.cache.values()]self.cache.clear()+ _note_dropped_pools(dropped)_drop_pool_if_unused()return self@@ -1130,11 +1286,15 @@ class GraphedForward:type(self.module).__name__,reason,)- if self.cache.pop(key, None) is not None:+ dropped_entry = self.cache.pop(key, None)+ if dropped_entry is not None:try:_torch().cuda.current_stream().synchronize() # no replay still running on what is freedexcept Exception: # noqa: BLE001pass+ gone = getattr(dropped_entry, "pool_token", None)+ del dropped_entry # counted once the graph is gone, never before+ _note_dropped_pools([gone])_drop_pool_if_unused()if not self.cache:self.capture_error = {"type": "Refused", "msg": reason}@@ -1268,7 +1428,9 @@ class GraphedForward:self.stats["invalidations"] += 1# Not reset(): its empty_cache on every model-offload onload makes the next onload pay the# cudaMallocs again. The dropped pool's blocks stay with the allocator.+ dropped = [getattr(e, "pool_token", None) for e in self.cache.values()]self.cache.clear()+ _note_dropped_pools(dropped) # the next capture flushes it before it records_drop_pool_if_unused() # the next capture must not record into a pool no graph holdsself._warmed.clear()self.valid_token = token@@ -1388,8 +1550,9 @@ class GraphedForward:return call(*warm_args, **warm_kwargs)if len(self.cache) >= self.max_graphs and self.plan is not None:# A planned call keys on the prompt length, so evict the least recently replayed graph.- self.cache.pop(next(iter(self.cache)))+ gone = getattr(self.cache.pop(next(iter(self.cache))), "pool_token", None)self.stats["evictions"] += 1+ _note_dropped_pools([gone])# The last graph on the shared pool takes the pool with it; never hand its id to the next capture._drop_pool_if_unused()if len(self.cache) >= self.max_graphs:@@ -1639,6 +1802,7 @@ class GraphedForward:pool = torch.cuda.graph_pool_handle()recorder = _graph_without_flush if recapture else torch.cuda.graphtry:+ # _capturing hands back any retired pool first, so the no-flush re-record never records beside one.with _capturing(), recorder(graph, pool = pool):out = record(*static_args, **static_kwargs)except BaseException as exc:diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex ffae0f233e8..57883d7982b 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -4189,10 +4189,8 @@ def install_encode_release(restore = release_resident_groups(owner, surplus, logger, denoisers_only = True, reason = "the prompt encode")+ # No empty_cache: the allocator frees cached segments and retries on OOM; flushing per prompt cost a sync.pending.append(restore)- if restore is not None and torch.cuda.is_available():- # freed blocks sit in the default stream's pool; the encoder onloads on the copy stream- torch.cuda.empty_cache()def _after(module: Any, args: Any, output: Any) -> None:if not pending:diff --git a/studio/backend/core/inference/diffusion_offload_prefetch.py b/studio/backend/core/inference/diffusion_offload_prefetch.pyindex 69ead084a71..1774c2396f4 100644--- a/studio/backend/core/inference/diffusion_offload_prefetch.py+++ b/studio/backend/core/inference/diffusion_offload_prefetch.py@@ -620,9 +620,7 @@ class GroupPrefetcher:self._drop_ring()def _drop_ring(self) -> None:- import torch-- from .diffusion_cuda_graph import hold_off_capture+ from .diffusion_cuda_graph import flush_now_locked, hold_off_capturewith hold_off_capture() as safe:if not safe:# Another thread is recording: its capture prohibits these syncs and frees. Retry after a later forward.@@ -630,16 +628,9 @@ class GroupPrefetcher:returnself._drop_slots_at_end = Falseself.disable_slots()- try:- import gc-- gc.collect() # a declined step graph's recordings, whose pool is flushed below with the ring- clear = getattr(torch._C, "_cuda_clearCublasWorkspaces", None)- if callable(clear):- clear() # and the workspace cuBLAS made for their capture stream- torch.cuda.empty_cache() # the ring's segments go back to the device, not just to the cache- except Exception: # noqa: BLE001- pass+ # The ring's segments go back to the device, not just to the cache, with a declined step graph's+ # recordings (their pool, and the workspace cuBLAS made for their capture stream): acknowledged as flushed.+ flush_now_locked(clear_cublas = True)def disable_slots(self) -> None:"""Drop the ring. Only between forwards: every streamed group is back on the host by then."""diff --git a/studio/backend/tests/test_diffusion_block_graph.py b/studio/backend/tests/test_diffusion_block_graph.pyindex 05260c524d4..f089def3f92 100644--- a/studio/backend/tests/test_diffusion_block_graph.py+++ b/studio/backend/tests/test_diffusion_block_graph.py@@ -863,6 +863,139 @@ def test_compiled_streamed_blocks_record_below_their_hooks_with_no_graph_break()handle.free()+def _reset_retired(monkeypatch):+ if hasattr(cg, "_RETIRED_POOLS"):+ monkeypatch.setattr(cg, "_RETIRED_POOLS", [0, 0])+ else:+ monkeypatch.setattr(cg, "_RETIRED_POOL_DUE", [False], raising = False)+++def test_a_block_capture_after_a_failed_one_flushes_the_retired_pool_first(monkeypatch):+ """A failed per-block capture retires its pool. When it cannot be flushed then (another thread started recording,+ which prohibits the flush), the pool stays due: the next capture hands it back before it records, so it never+ records beside the retired segments. Every block still runs its compute exactly."""+ _cuda()+ _reset_retired(monkeypatch)+ net = _net(blocks = 2).cuda()+ ref = copy.deepcopy(net)+ handle, _ = bg.install_block_graphs(net, device = "cuda", slots = False)+ monkeypatch.setattr(cg, "_FAILED_GRAPHS", [])+ events = []+ real_flush = torch.cuda.empty_cache+ monkeypatch.setattr(torch.cuda, "empty_cache", lambda: events.append("flush") or real_flush())+ real_begin = torch.cuda.CUDAGraph.capture_begin+ other = {"recording": False}++ def begin(self, *args, **kwargs):+ if "fail" not in events:+ events.append("fail")+ cg._CAPTURE_DEPTH += 1 # another render thread starts recording as this capture fails+ other["recording"] = True+ raise RuntimeError("capture failed")+ events.append("begin")+ return real_begin(self, *args, **kwargs)++ real_refuse = cg.refuse_if_exhausted++ def refuse(*args, **kwargs):+ if other["recording"]: # ... and is done before the next block records+ other["recording"] = False+ cg._CAPTURE_DEPTH -= 1+ return real_refuse(*args, **kwargs)++ monkeypatch.setattr(torch.cuda.CUDAGraph, "capture_begin", begin)+ monkeypatch.setattr(cg, "refuse_if_exhausted", refuse)+ x = torch.randn(7, 16, device = "cuda")+ try:+ with torch.inference_mode():+ for _ in range(4):+ assert torch.equal(net(x), ref(x))+ finally:+ if other["recording"]:+ cg._CAPTURE_DEPTH -= 1+ start = events.index("fail")+ assert "begin" in events[start:]+ assert "flush" in events[start : events.index("begin", start)]+ if hasattr(cg, "retired_pool_due"):+ assert not cg.retired_pool_due()+ handle.free()+++def test_a_block_poisoned_before_capture_counts_the_graphs_it_dropped(monkeypatch):+ """A block with recorded layouts that fails before capture_begin on a new one (budget, warm-up, statics) is+ poisoned and drops every graph it held: their pool counts as retired, so the next capture flushes it first."""+ _cuda()+ _reset_retired(monkeypatch)+ net = _net(blocks = 2).cuda()+ ref = copy.deepcopy(net)+ handle, _ = bg.install_block_graphs(net, device = "cuda", slots = False)+ x = torch.randn(7, 16, device = "cuda")+ with torch.inference_mode():+ for _ in range(4):+ assert torch.equal(net(x), ref(x))+ held = sum(len(g.cache) for g in handle.graphs)+ assert held >= 1++ def refuse(*args, **kwargs):+ raise RuntimeError("no room for the statics")++ monkeypatch.setattr(cg, "refuse_if_exhausted", refuse)+ y = torch.randn(5, 16, device = "cuda")+ with torch.inference_mode():+ for _ in range(4):+ assert torch.equal(net(y), ref(y))+ assert any(g.poisoned for g in handle.graphs)+ due = cg.retired_pool_due() if hasattr(cg, "retired_pool_due") else False+ assert due+ handle.free()+++def test_evicting_a_block_graph_from_a_retired_pool_counts_it(monkeypatch):+ """After a failed capture replaced the shared pool, older recordings still hold the old one. When the LRU evicts+ the last of them, that pool retires: it is counted, so the next capture flushes it before recording beside it."""+ _cuda()+ _reset_retired(monkeypatch)+ net = _net(blocks = 1).cuda()+ ref = copy.deepcopy(net)+ handle, _ = bg.install_block_graphs(net, device = "cuda", slots = False)+ block = handle.graphs[0]+ limit = block.max_graphs+ with torch.inference_mode():+ for rows in range(2, 2 + limit):+ x = torch.randn(rows, 16, device = "cuda")+ for _ in range(3):+ assert torch.equal(net(x), ref(x))+ assert len(block.cache) == limit+ block.shared.pool = None # what retire_failed_capture leaves: later recordings get a fresh pool+ _reset_retired(monkeypatch)+ with torch.inference_mode():+ x = torch.randn(2 + limit, 16, device = "cuda")+ for _ in range(3):+ assert torch.equal(net(x), ref(x))+ assert block.stats["evictions"] >= 1+ due = cg.retired_pool_due() if hasattr(cg, "retired_pool_due") else False+ assert due+ handle.free()+++def test_a_block_capture_refused_before_recording_flushes_nothing(monkeypatch):+ """Past the failure budget every block fails before capture_begin: no pool was retired, so no allocator flush."""+ _cuda()+ _reset_retired(monkeypatch)+ net = _net(blocks = 2).cuda()+ ref = copy.deepcopy(net)+ handle, _ = bg.install_block_graphs(net, device = "cuda", slots = False)+ flushes = cg._POOL_FLUSHES[0]+ monkeypatch.setattr(cg, "captures_exhausted", lambda: True)+ x = torch.randn(7, 16, device = "cuda")+ with torch.inference_mode():+ for _ in range(3):+ assert torch.equal(net(x), ref(x))+ assert handle.stats["fallbacks"] + handle.stats.get("segment_fallbacks", 0) >= 1+ assert cg._POOL_FLUSHES[0] == flushes and not cg.retired_pool_due()+ handle.free()++def test_alternating_layouts_both_record():"""True CFG calls each block with two text lengths in turn; each layout must still record and replay."""_cuda()diff --git a/studio/backend/tests/test_diffusion_measured_placement.py b/studio/backend/tests/test_diffusion_measured_placement.pyindex 4559f6c9f2b..145b676a9a3 100644--- a/studio/backend/tests/test_diffusion_measured_placement.py+++ b/studio/backend/tests/test_diffusion_measured_placement.py@@ -807,6 +807,29 @@ def test_encode_streams_the_whole_resident_denoiser_back_to_the_flat_room(monkeyassert torch.equal(net(x.cuda()).cpu(), ref)+def test_encode_release_does_not_flush_the_allocator_per_prompt(monkeypatch):+ """The encode release streams groups and pins them back on every new prompt. It leaves the freed blocks cached:+ an encoder allocation that needs the room gets it back from the allocator itself. An empty_cache per prompt cost a+ device sync plus fresh cudaMallocs for the encoder on every render."""+ torch, net, enc, pipe, x, ref, on_device = _encode_release_pipe(monkeypatch)+ block_mib = 4+ assert dm.install_encode_release(pipe, _whole_plan(1024, 1024 - 2 * block_mib), None) == 1+ flushes = {"n": 0}+ real = torch.cuda.empty_cache++ def counted():+ flushes["n"] += 1+ real()++ monkeypatch.setattr(torch.cuda, "empty_cache", counted)+ for _ in range(3):+ enc(torch.randn(2, 8, device = "cuda"))+ assert enc.seen[-1] == ["cuda"] * 4 + ["cpu"] * 2 # the release still ran+ assert on_device() == ["cuda"] * 6+ assert torch.equal(net(x.cuda()).cpu(), ref)+ assert flushes["n"] == 0++def test_encode_release_restores_after_a_failed_encode(monkeypatch):torch, net, enc, pipe, x, ref, on_device = _encode_release_pipe(monkeypatch)dm.install_encode_release(pipe, _whole_plan(1024, 0), None)diff --git a/studio/backend/tests/test_diffusion_offload_cuda_graph.py b/studio/backend/tests/test_diffusion_offload_cuda_graph.pyindex b13d1d47050..13b5897543f 100644--- a/studio/backend/tests/test_diffusion_offload_cuda_graph.py+++ b/studio/backend/tests/test_diffusion_offload_cuda_graph.py@@ -167,6 +167,394 @@ def test_release_and_restore_records_again_and_stays_exact():assert handles[0].stats["invalidations"] >= 1 and handles[0].stats["captures"] >= 2+def _reset_retired(monkeypatch):+ if hasattr(cg, "_RETIRED_POOLS"):+ monkeypatch.setattr(cg, "_RETIRED_POOLS", [0, 0])+ else:+ monkeypatch.setattr(cg, "_RETIRED_POOL_DUE", [False], raising = False)+++def _retire():+ """Another wrapper's invalidation retired a pool."""+ if hasattr(cg, "note_pool_retired"):+ cg.note_pool_retired()+ else:+ cg._RETIRED_POOL_DUE[0] = True+++def _due():+ if hasattr(cg, "retired_pool_due"):+ return cg.retired_pool_due()+ return cg._RETIRED_POOL_DUE[0]+++def test_an_offloaded_re_record_hands_the_dropped_pool_back_first(monkeypatch):+ """Each new prompt encode moves the resident groups (release, the encoder takes the room, re-pin elsewhere), so+ the step records again. The encode release no longer flushes the allocator; the re-record itself hands the+ dropped recording's pool back before it records, so the reserved memory stays flat instead of growing by a pool+ per prompt, and every output stays exact."""+ _cuda()+ _reset_retired(monkeypatch)+ seeds = list(range(6))+ want = _eager_reference(3, seeds)+ torch.manual_seed(0)+ net = _streamed(_Net(), 3)+ pipe = _pipe(net)+ handles, _ = cg.arm_after_placement(pipe)+ got = _run(net, seeds[:2])+ flushes = cg._POOL_FLUSHES[0]+ reserved = []+ for cycle in range(4):+ restore = dm.release_resident_groups(pipe, 10_000, None)+ hold = torch.empty(+ 16 << 20, dtype = torch.uint8, device = "cuda"+ ) # the encoder takes the freed room+ restore()+ del hold+ got += _run(net, seeds[2 + cycle : 3 + cycle])+ reserved.append(torch.cuda.memory_reserved())+ for a, b in zip(got, want):+ assert torch.equal(a, b)+ records = handles[0].stats["captures"]+ assert records >= 2 # the step recorded again after the moves+ assert cg._POOL_FLUSHES[0] - flushes >= records - 1+ assert reserved[-1] <= reserved[0]+ cg.uninstall_all(handles)+++def _two_shapes(net, shapes = (16, 8)):+ outs = []+ with torch.no_grad():+ for rows in shapes:+ g = torch.Generator(device = "cpu").manual_seed(rows)+ x = torch.randn(rows, 64, generator = g).cuda()+ t = torch.rand(1, generator = g).cuda()+ outs.append(net(x, t, return_dict = False)[0].clone())+ torch.cuda.synchronize()+ return outs+++def test_one_invalidation_flushes_the_retired_pool_once_for_every_re_recorded_key(monkeypatch):+ """Two keys re-recorded under one new placement share one retired pool: the first re-record flushes it, the second+ does not flush the allocator again."""+ _cuda()+ _reset_retired(monkeypatch)+ torch.manual_seed(0)+ net = _streamed(_Net(), 3)+ pipe = _pipe(net)+ handles, _ = cg.arm_after_placement(pipe)+ _two_shapes(net)+ _two_shapes(net)+ assert handles[0].stats["captures"] == 2+ flushes = cg._POOL_FLUSHES[0]+ for cycle in range(2):+ restore = dm.release_resident_groups(pipe, 10_000, None)+ hold = torch.empty(16 << 20, dtype = torch.uint8, device = "cuda")+ restore()+ del hold+ _two_shapes(net)+ _two_shapes(net)+ assert cg._POOL_FLUSHES[0] - flushes == cycle + 1+ assert handles[0].stats["captures"] == 6+ cg.uninstall_all(handles)+++def test_a_re_record_waits_for_the_retired_pool_flush(monkeypatch):+ """While the flush is prohibited (another thread records), the re-record is deferred and the step runs eager; it+ records once that recording ended and the flush went through."""+ _cuda()+ _reset_retired(monkeypatch)+ monkeypatch.setattr(cg, "RETIRED_FLUSH_WAIT_S", 0.01)+ seeds = list(range(5))+ want = _eager_reference(3, seeds)+ torch.manual_seed(0)+ net = _streamed(_Net(), 3)+ pipe = _pipe(net)+ handles, _ = cg.arm_after_placement(pipe)+ got = _run(net, seeds[:2])+ assert handles[0].stats["captures"] == 1+ restore = dm.release_resident_groups(pipe, 10_000, None)+ hold = torch.empty(16 << 20, dtype = torch.uint8, device = "cuda")+ restore()+ del hold+ monkeypatch.setattr(cg, "_CAPTURE_DEPTH", 1) # another thread is recording+ got += _run(net, seeds[2:3])+ assert handles[0].stats["captures"] == 1+ assert handles[0].stats.get("capture_refusals", 0) >= 1 and cg.retired_pool_due()+ monkeypatch.setattr(cg, "_CAPTURE_DEPTH", 0)+ got += _run(net, seeds[3:])+ assert handles[0].stats["captures"] == 2 and not cg.retired_pool_due()+ for a, b in zip(got, want):+ assert torch.equal(a, b)+ cg.uninstall_all(handles)+++def test_a_capture_refused_while_another_records_never_empties_the_cache(monkeypatch):+ """A capture refused because another thread is recording (a retired pool is due and cannot be flushed) runs eager+ without touching the allocator: an empty_cache or a cuBLAS workspace clear inside that recording could invalidate+ it. The cleanup stays due and the next capture's entry flush does it once the recording ended."""+ _cuda()+ _reset_retired(monkeypatch)+ monkeypatch.setattr(cg, "RETIRED_FLUSH_WAIT_S", 0.01)+ torch.manual_seed(0)+ net = _streamed(_Net(), 3)+ handles, _ = cg.arm_after_placement(_pipe(net))+ _two_shapes(net)+ before = handles[0].stats["captures"]+ _retire() # another wrapper retired a pool+ real = torch.cuda.empty_cache+ during = []+ monkeypatch.setattr(+ torch.cuda, "empty_cache", lambda: during.append(cg._CAPTURE_DEPTH) or real()+ )+ monkeypatch.setattr(cg, "_CAPTURE_DEPTH", 1) # another thread is recording+ _two_shapes(net, (12, 4)) # new keys: each capture is refused at its entry+ monkeypatch.setattr(cg, "_CAPTURE_DEPTH", 0)+ assert handles[0].stats["captures"] == before+ assert handles[0].stats.get("capture_refusals", 0) >= 1+ assert during == [] # nothing flushed while the other capture recorded+ _two_shapes(net, (12, 4))+ assert handles[0].stats["captures"] == before + 2 and not _due()+ cg.uninstall_all(handles)+++def test_a_deferred_release_still_clears_the_cublas_workspaces_before_the_next_recording(+ monkeypatch,+):+ """An offloaded wrapper's release also clears the cuBLAS workspaces its recordings left on their capture stream.+ Deferred because another thread was recording, that clear is owed with the retired pool: the next capture's entry+ flush does it under the capture lock (no capture recording), once, before it records."""+ _cuda()+ _reset_retired(monkeypatch)+ monkeypatch.setattr(cg, "_CUBLAS_CLEAR_DUE", [False], raising = False)+ monkeypatch.setattr(cg, "RETIRED_FLUSH_WAIT_S", 0.01)+ torch.manual_seed(0)+ net = _streamed(_Net(), 3)+ handles, _ = cg.arm_after_placement(_pipe(net))+ _two_shapes(net)+ before = handles[0].stats["captures"]+ _retire()+ real = getattr(torch._C, "_cuda_clearCublasWorkspaces")+ clears = []++ def clear():+ clears.append(cg._CAPTURE_DEPTH)+ return real()++ monkeypatch.setattr(torch._C, "_cuda_clearCublasWorkspaces", clear)+ monkeypatch.setattr(cg, "_CAPTURE_DEPTH", 1) # another thread is recording+ _two_shapes(net, (12, 4)) # refused at entry: the release after it is deferred+ monkeypatch.setattr(cg, "_CAPTURE_DEPTH", 0)+ assert handles[0].stats["captures"] == before and clears == []+ _two_shapes(net, (12, 4))+ assert handles[0].stats["captures"] == before + 2+ assert clears == [0] # once, outside any recording+ assert not cg._CUBLAS_CLEAR_DUE[0] and not _due()+ cg.uninstall_all(handles)+++def test_a_settled_retirement_never_waits_on_another_capture(monkeypatch):+ """Two threads saw the same retirement; the first flushed it and is now recording. The second must not wait on+ that recording (and be refused after the timeout): its retirements are already flushed."""+ monkeypatch.setattr(cg, "_RETIRED_POOLS", [1, 1], raising = False)+ monkeypatch.setattr(cg, "RETIRED_FLUSH_WAIT_S", 5.0)+ monkeypatch.setattr(cg, "_CAPTURE_DEPTH", 1) # the first thread is recording+ import time++ start = time.monotonic()+ with cg._CAPTURE_IDLE:+ cg._settle_retired_pools(1)+ assert time.monotonic() - start < 1.0+++def test_an_immediate_release_acknowledges_the_retirement_it_flushed(monkeypatch):+ """A speed drop or a reset counts the dropped pool and then releases it at once (no capture recording): that flush+ acknowledges it, so the next capture does not synchronize and flush the allocator a second time."""+ _cuda()+ _reset_retired(monkeypatch)+ torch.manual_seed(0)+ net = torch.nn.Linear(64, 64).cuda()+ handle = cg.GraphedForward(net)+ _retire() # the drop just counted its pool+ handle._release() # ... and released it right away+ assert not _due()+++def test_a_capture_with_a_pool_due_waits_out_another_recording_instead_of_running_eager(+ monkeypatch,+):+ """PyTorch runs no non-captured CUDA work on any thread while a capture is underway. A capture whose retired pool+ must be flushed while another thread records therefore waits for that recording to end, then flushes and records;+ it is not refused into an eager step that would run beside the recording."""+ import threading+ import time++ monkeypatch.setattr(cg, "_RETIRED_POOLS", [1, 0], raising = False)+ flushed = []+ monkeypatch.setattr(+ cg, "_flush_retired_locked", lambda seen: flushed.append(cg._CAPTURE_DEPTH) or True+ )+ monkeypatch.setattr(cg, "_CAPTURE_DEPTH", 1) # another thread is recording++ def other_recording_ends():+ time.sleep(0.6)+ with cg._CAPTURE_IDLE:+ cg._CAPTURE_DEPTH = 0+ cg._CAPTURE_IDLE.notify_all()++ ender = threading.Thread(target = other_recording_ends)+ ender.start()+ try:+ with cg._CAPTURE_IDLE:+ cg._settle_retired_pools(1)+ finally:+ ender.join()+ assert flushed == [0]+++def test_a_deferred_release_is_owed_before_it_tries_the_lock(monkeypatch):+ """The release counts its pool (and the cuBLAS clear) before gc and the lock attempt: a capture entering meanwhile+ already sees it due instead of recording beside the unflushed segments."""+ _reset_retired(monkeypatch)+ monkeypatch.setattr(cg, "_CUBLAS_CLEAR_DUE", [False], raising = False)+ seen_during_gc = []++ class _Gc:+ @staticmethod+ def collect():+ seen_during_gc.append((cg.retired_pool_due(), cg._CUBLAS_CLEAR_DUE[0]))++ @__import__("contextlib").contextmanager+ def recording_elsewhere():+ yield False++ monkeypatch.setattr(cg, "gc", _Gc)+ monkeypatch.setattr(cg, "hold_off_capture", recording_elsewhere)+ assert cg.release_cached_outside_capture(clear_cublas = True) is False+ assert seen_during_gc == [(True, True)]+ assert cg.retired_pool_due() and cg._CUBLAS_CLEAR_DUE[0]+++def test_a_pool_retired_while_a_capture_enters_is_flushed_before_it_records(monkeypatch):+ """Another thread retires a pool after this capture read the counters but before it raised the depth (here, while+ it waits for replica-side work to drain): the entry flushes that one too, so nothing records while it is due."""+ monkeypatch.setattr(cg, "_RETIRED_POOLS", [1, 0], raising = False)+ flushes = []++ def flush(seen):+ flushes.append(seen)+ cg._RETIRED_POOLS[1] = max(cg._RETIRED_POOLS[1], seen)+ return True++ late = []++ def drained(own):+ if not late:+ late.append(1)+ cg.note_pool_retired() # lands after the entry flush, before the depth goes up++ monkeypatch.setattr(cg, "_flush_retired_locked", flush)+ monkeypatch.setattr(cg, "_wait_outside_drained", drained)+ monkeypatch.setattr(cg.gc, "collect", lambda *a: 0)+ with cg._capturing():+ due_while_recording = cg.retired_pool_due()+ assert late and not due_while_recording+ assert flushes == [1, 2]+++def test_a_first_capture_refused_before_entry_keeps_the_retired_pool_due(monkeypatch):+ """Another wrapper retired a pool; this wrapper's first capture is refused in _capturing (replica-side work still+ held) before anything flushes: the flush stays due for the next capture."""+ _cuda()+ _reset_retired(monkeypatch)+ torch.manual_seed(0)+ net = _streamed(_Net(), 3)+ handles, _ = cg.arm_after_placement(_pipe(net))+ _retire()+ monkeypatch.setattr(cg, "capture_refused_now", lambda: False)++ def refused(own):+ raise cg.CaptureRefused("replica-side CUDA work still running; capture refused")++ monkeypatch.setattr(cg, "_wait_outside_drained", refused)+ monkeypatch.setattr(cg, "await_outside_drained", lambda: None)+ monkeypatch.setattr(cg, "RETIRED_FLUSH_WAIT_S", 0.01)+ monkeypatch.setattr(cg, "_CAPTURE_DEPTH", 1) # the flush cannot run either+ _run(net, [0, 1])+ monkeypatch.setattr(cg, "_CAPTURE_DEPTH", 0)+ assert handles[0].stats["captures"] == 0+ assert handles[0].stats.get("capture_refusals", 0) >= 1+ assert _due()+ cg.uninstall_all(handles)+++def test_a_pool_retired_while_the_capture_flushes_is_flushed_before_it_records(monkeypatch):+ """Retirement is not serialized with captures: another thread's invalidation can retire a pool while this+ capture's entry flush runs. That flush acknowledges only what was retired before it began, and the entry rechecks+ before it raises the depth, so the newly retired pool is flushed too before anything records."""+ _cuda()+ _reset_retired(monkeypatch)+ torch.manual_seed(0)+ net = _streamed(_Net(), 3)+ handles, _ = cg.arm_after_placement(_pipe(net))+ _retire()+ real = torch.cuda.empty_cache+ raced = []+ due_at_record = []++ def flush():+ real()+ if not raced:+ raced.append(1)+ _retire() # lands after this flush freed what it could, before the capture records++ real_begin = torch.cuda.CUDAGraph.capture_begin++ def begin(self, *args, **kwargs):+ due_at_record.append(_due())+ return real_begin(self, *args, **kwargs)++ monkeypatch.setattr(torch.cuda, "empty_cache", flush)+ monkeypatch.setattr(torch.cuda.CUDAGraph, "capture_begin", begin)+ flushes = cg._POOL_FLUSHES[0]+ _run(net, [0, 1])+ assert raced and handles[0].stats["captures"] >= 1+ assert due_at_record and not any(due_at_record)+ assert cg._POOL_FLUSHES[0] - flushes == 2 and not _due()+ cg.uninstall_all(handles)+++def test_a_resident_planned_re_record_flushes_a_pool_another_wrapper_retired(monkeypatch):+ """The retired-pool rule is process-wide, not per offload placement: a resident wrapper with a capture-safe plan+ re-records later keys without torch.cuda.graph's flush, so a pool an offloaded wrapper retired must be handed back+ before that re-record too, or it records a new prompt shape beside the reserved segments."""+ _cuda()+ _reset_retired(monkeypatch)+ torch.manual_seed(0)+ net = torch.nn.Linear(64, 64).cuda()+ handle = cg.GraphedForward(net)+ handle.plan = (+ lambda *a: None+ ) # a planned step: keys after the first re-record with no flush of their own+ real = torch.cuda.empty_cache+ flushes = []+ with torch.no_grad():+ first = handle._capture((torch.randn(16, 64, device = "cuda"),), {}, key = "a")+ _retire()+ monkeypatch.setattr(torch.cuda, "empty_cache", lambda: flushes.append(1) or real())+ x = torch.randn(8, 64, device = "cuda")+ second = handle._capture((x,), {}, key = "b")+ second.static[0].copy_(x)+ second.graph.replay()+ torch.cuda.synchronize()+ out = second.out_tensors[0].clone()+ assert handle.stats["captures"] == 2+ assert flushes and not _due()+ assert torch.equal(out, net(x))+ del first, second+ cg._POOL_BOX[0] = None++def test_unpinned_host_copies_keep_the_forward_eager_with_the_reason(monkeypatch):_cuda()monkeypatch.setenv("UNSLOTH_DIFFUSION_GROUP_OFFLOAD_PIN", "0")