Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: do not subtract live segment graph outputs twice from reusable cache (#16231)

Daniel Hanbabbdbf9462f
diff --git a/studio/backend/core/inference/diffusion_block_graph.py b/studio/backend/core/inference/diffusion_block_graph.py
index 2789498ae96..ae8798949cf 100644
--- a/studio/backend/core/inference/diffusion_block_graph.py
+++ b/studio/backend/core/inference/diffusion_block_graph.py
@@ -250,15 +250,35 @@ _LIVE: "weakref.WeakSet" = weakref.WeakSet()
def pool_bytes(device: Optional[int] = None) -> int:
- """Device bytes held in every live block-graph pool (on CUDA ``device`` when given): reserved by the allocator yet
- never free for other work, so the memory guard must not credit them back as reclaimable."""
+ """Unused graph-private bytes; retained outputs already count as allocated."""
total = 0
+ tracked = 0
+ pools: set = set()
for shared in tuple(_LIVE):
index = getattr(shared, "device_index", None)
if device is not None and index is not None and index != device:
continue
- total += int(getattr(shared, "pool_bytes", 0) or 0)
- return total
+ recorded = int(getattr(shared, "pool_bytes", 0) or 0)
+ handles = getattr(shared, "pools", None)
+ if handles:
+ tracked += recorded
+ pools.update(tuple(h) for h in handles)
+ else:
+ total += recorded
+ if not pools:
+ return total
+ try:
+ free = 0
+ from .diffusion_cuda_graph import _allocator_segments
+
+ for seg in _allocator_segments():
+ if device is not None and seg.get("device", device) != device:
+ continue
+ if tuple(seg.get("segment_pool_id") or ()) in pools:
+ free += int(seg.get("total_size", 0)) - int(seg.get("allocated_size", 0))
+ return total + max(0, free)
+ except Exception: # noqa: BLE001
+ return total + tracked
class _Shared:
@@ -278,6 +298,7 @@ class _Shared:
self.static_out: dict = {}
self.static_refs: dict = {} # slot -> recordings reading its buffers
self.warmed: set = set()
+ self.pools: list = [] # Includes retired pools.
self.pool_bytes = 0
self.static_bytes = 0
self.strict_device = False
@@ -293,6 +314,7 @@ class _Shared:
if self.pool is None:
torch = _torch()
self.pool = torch.cuda.graph_pool_handle()
+ self.pools.append(self.pool)
return self.pool
def statics_for(self, slot: tuple, live: list) -> list:
@@ -337,6 +359,7 @@ class _Shared:
self.static_refs.clear()
self.warmed.clear()
self.pool = None
+ self.pools = []
self.pool_bytes = 0
self.static_bytes = 0
@@ -587,7 +610,6 @@ class BlockGraph:
dst.copy_(src)
static_args, static_kwargs = _unwalk(key, iter(entry.static_in))
if slot not in self.shared.warmed:
- # Any guard the static buffers trip recompiles here, never inside the recording.
self.compute(*static_args, **static_kwargs)
self.shared.warmed.add(slot)
self.stats["warmups"] += 1
@@ -599,7 +621,6 @@ class BlockGraph:
pool = self.shared.graph_pool()
with torch.cuda.stream(stream):
with _capturing():
- # read after the entry, which first hands back any retired graph pool
before = torch.cuda.memory_reserved()
try:
# Inside the try: a capture_begin that raised after taking the pool is abandoned too.
@@ -620,9 +641,10 @@ class BlockGraph:
if self.shared.pool == pool:
self.shared.pool = None
raise
+ after = torch.cuda.memory_reserved()
del out, flat
current.wait_stream(stream)
- self.shared.pool_bytes += max(0, torch.cuda.memory_reserved() - before)
+ self.shared.pool_bytes += max(0, after - before)
except CaptureRefused:
# Refused before capture_begin: nothing recorded, the pool untouched. Eager now, record on a later call.
self.seen[full] = (out_spec, metas)
@@ -1043,7 +1065,6 @@ class SegmentGraph:
pool = self.shared.graph_pool()
with torch.cuda.stream(stream):
with _capturing():
- # read after the entry, which first hands back any retired graph pool
before = torch.cuda.memory_reserved()
try:
graph.capture_begin(pool = pool, capture_error_mode = "thread_local")
@@ -1056,8 +1077,9 @@ class SegmentGraph:
if self.shared.pool == pool:
self.shared.pool = None
raise
+ after = torch.cuda.memory_reserved()
current.wait_stream(stream)
- self.shared.pool_bytes += max(0, torch.cuda.memory_reserved() - before)
+ self.shared.pool_bytes += max(0, after - before)
except CaptureRefused:
self.seen[full] = rec
self.stats["segment_refusals"] = self.stats.get("segment_refusals", 0) + 1
diff --git a/studio/backend/core/inference/diffusion_cuda_graph.py b/studio/backend/core/inference/diffusion_cuda_graph.py
index 351b1673ea0..c992a179530 100644
--- a/studio/backend/core/inference/diffusion_cuda_graph.py
+++ b/studio/backend/core/inference/diffusion_cuda_graph.py
@@ -899,12 +899,25 @@ def _prewarm_token_counts(live: list) -> tuple:
return tuple(sorted(counts)[:8])
+def _allocator_segments() -> list:
+ torch = _torch()
+ native = getattr(getattr(torch, "_C", None), "_cuda_memorySnapshot", None)
+ if native is None:
+ return torch.cuda.memory_snapshot()
+ # Avoid borrowed-reference corruption in the tuple API: pytorch/pytorch#201043.
+ try:
+ snapshot = native(None)
+ except TypeError:
+ snapshot = native()
+ return snapshot["segments"]
+
+
def _pool_bytes(graph: Any) -> int:
"""Device bytes the graph's private pool holds (shared by every graph of the load), 0 if unreadable."""
try:
pool = tuple(graph.pool())
total = 0
- for seg in _torch().cuda.memory_snapshot():
+ for seg in _allocator_segments():
if tuple(seg.get("segment_pool_id") or ()) == pool:
total += int(seg.get("total_size", 0))
return total
@@ -1012,8 +1025,8 @@ def live_pool_free_bytes() -> int:
try:
free = 0
cuda = _torch().cuda
- device = cuda.current_device() # the snapshot spans every card
- for seg in cuda.memory_snapshot():
+ device = cuda.current_device()
+ for seg in _allocator_segments():
if seg.get("device", device) != device:
continue
if tuple(seg.get("segment_pool_id") or ()) in pools:
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index 98e8cb725b6..c20d520c15d 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -1239,9 +1239,6 @@ def _metal_working_set_mib() -> tuple[Optional[int], int]:
def _graph_held_cache_bytes(device: Optional[int] = None) -> int:
- """Bytes inside ``memory_reserved() - memory_allocated()`` that ordinary allocations cannot reuse: the CUDA graph
- pools (per-block pools whole, the whole-step graphs' shared pool's unallocated part). ``device`` None counts every
- block pool and reads the whole-step pool on the current card."""
held = 0
try:
from .diffusion_block_graph import pool_bytes as _block_graph_pool_bytes
diff --git a/studio/backend/tests/test_diffusion_block_graph.py b/studio/backend/tests/test_diffusion_block_graph.py
index f089def3f92..55382b8868d 100644
--- a/studio/backend/tests/test_diffusion_block_graph.py
+++ b/studio/backend/tests/test_diffusion_block_graph.py
@@ -6,6 +6,7 @@
from __future__ import annotations
import copy
+import contextlib
import types
import warnings
@@ -714,6 +715,96 @@ def test_pool_bytes_count_only_the_target_card(monkeypatch):
assert bg.pool_bytes(0) == 100 and bg.pool_bytes() == 1000
[email protected]("legacy", [False, True])
+def test_allocator_snapshot_avoids_tuple_arguments(monkeypatch, legacy):
+ calls = []
+ segments = [{"total_size": 123}]
+
+ def native(*args):
+ calls.append(args)
+ if legacy and args:
+ raise TypeError("takes no arguments")
+ return {"segments": segments}
+
+ fake = types.SimpleNamespace(_C = types.SimpleNamespace(_cuda_memorySnapshot = native))
+ monkeypatch.setattr(cg, "_torch", lambda: fake)
+ assert cg._allocator_segments() is segments
+ assert calls == ([(None,), ()] if legacy else [(None,)])
+
+
+def test_known_pool_is_counted_when_net_reservation_is_zero(monkeypatch):
+ shared = bg._Shared(0)
+ shared.pools.append((7, 1))
+ monkeypatch.setattr(bg, "_LIVE", {shared})
+ segments = [{"device": 0, "segment_pool_id": (7, 1), "total_size": 200, "allocated_size": 80}]
+ monkeypatch.setattr(torch.cuda, "memory_snapshot", lambda: segments)
+ monkeypatch.setattr(
+ torch._C, "_cuda_memorySnapshot", lambda *a: {"segments": segments}, raising = False
+ )
+ assert shared.pool_bytes == 0
+ assert bg.pool_bytes(0) == 120
+
+
+def test_capture_reservation_precedes_deferred_cache_release(monkeypatch):
+ _cuda()
+ original = cg._capturing
+
+ @contextlib.contextmanager
+ def capturing():
+ with original():
+ yield
+ cg.release_when_capture_idle(torch.cuda.empty_cache)
+
+ monkeypatch.setattr(cg, "_capturing", capturing)
+ block = Block().cuda()
+ shared = bg._Shared(torch.cuda.current_device(), None)
+ graph = bg.BlockGraph(block, block.forward, shared)
+ x = torch.randn(8192, 64, device = "cuda")
+ try:
+ with torch.inference_mode():
+ graph(x)
+ idle = torch.empty(128 << 20, dtype = torch.uint8, device = "cuda")
+ del idle
+ graph(x)
+ assert graph.stats["captures"] == 1
+ assert shared.pool_bytes > 0
+ assert torch.equal(graph(x), block(x))
+ finally:
+ graph.reset()
+ shared.release()
+
+
+def test_live_segment_outputs_in_a_graph_pool_are_not_subtracted_twice(monkeypatch):
+ mib = 1 << 20
+ shared = bg._Shared(0)
+ monkeypatch.setattr(bg, "_LIVE", {shared})
+ monkeypatch.setattr(torch.cuda, "graph_pool_handle", lambda: (7, 1))
+ shared.graph_pool()
+ shared.pool_bytes = 500 * mib
+ snapshot = [
+ {
+ "device": 0,
+ "segment_pool_id": (7, 1),
+ "total_size": 500 * mib,
+ "allocated_size": 500 * mib,
+ },
+ {"device": 0, "segment_pool_id": (0, 0), "total_size": 100 * mib, "allocated_size": 0},
+ ]
+ monkeypatch.setattr(torch.cuda, "memory_snapshot", lambda: snapshot)
+ monkeypatch.setattr(cg, "_allocator_segments", lambda: snapshot, raising = False)
+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda *a, **k: 600 * mib)
+ monkeypatch.setattr(torch.cuda, "memory_allocated", lambda *a, **k: 500 * mib)
+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)
+ monkeypatch.setattr(cg, "live_pool_free_bytes", lambda: 0)
+ assert dm.cuda_reusable_cache_bytes(0) == 100 * mib
+ snapshot[0]["allocated_size"] = 250 * mib
+ monkeypatch.setattr(torch.cuda, "memory_allocated", lambda *a, **k: 250 * mib)
+ assert dm.cuda_reusable_cache_bytes(0) == 100 * mib
+ monkeypatch.setattr(torch.cuda, "memory_snapshot", lambda: 1 / 0)
+ monkeypatch.setattr(cg, "_allocator_segments", lambda: 1 / 0, raising = False)
+ assert dm.cuda_reusable_cache_bytes(0) == 0
+
+
def test_block_graph_pools_are_not_credited_as_reclaimable(monkeypatch):
target = types.SimpleNamespace(device = "cuda", backend = "cuda")
snap = dm.DeviceMemory("cuda", "cuda", "dedicated", 1000, 8000)
diff --git a/studio/backend/tests/test_diffusion_offload_step_graphs.py b/studio/backend/tests/test_diffusion_offload_step_graphs.py
index e83c217c65f..bd3c40dc222 100644
--- a/studio/backend/tests/test_diffusion_offload_step_graphs.py
+++ b/studio/backend/tests/test_diffusion_offload_step_graphs.py
@@ -282,6 +282,7 @@ def test_every_live_graph_pool_stays_out_of_the_reclaimable_memory(monkeypatch):
},
]
monkeypatch.setattr(torch.cuda, "memory_snapshot", lambda: segments)
+ monkeypatch.setattr(cg, "_allocator_segments", lambda: segments, raising = False)
monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)
assert cg.live_pool_free_bytes() == 60