Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: do not subtract live segment graph outputs twice from reusable cache (#16231)
Daniel Hanbabbdbf9462f
diff --git a/studio/backend/core/inference/diffusion_block_graph.py b/studio/backend/core/inference/diffusion_block_graph.pyindex 2789498ae96..ae8798949cf 100644--- a/studio/backend/core/inference/diffusion_block_graph.py+++ b/studio/backend/core/inference/diffusion_block_graph.py@@ -250,15 +250,35 @@ _LIVE: "weakref.WeakSet" = weakref.WeakSet()def pool_bytes(device: Optional[int] = None) -> int:- """Device bytes held in every live block-graph pool (on CUDA ``device`` when given): reserved by the allocator yet- never free for other work, so the memory guard must not credit them back as reclaimable."""+ """Unused graph-private bytes; retained outputs already count as allocated."""total = 0+ tracked = 0+ pools: set = set()for shared in tuple(_LIVE):index = getattr(shared, "device_index", None)if device is not None and index is not None and index != device:continue- total += int(getattr(shared, "pool_bytes", 0) or 0)- return total+ recorded = int(getattr(shared, "pool_bytes", 0) or 0)+ handles = getattr(shared, "pools", None)+ if handles:+ tracked += recorded+ pools.update(tuple(h) for h in handles)+ else:+ total += recorded+ if not pools:+ return total+ try:+ free = 0+ from .diffusion_cuda_graph import _allocator_segments++ for seg in _allocator_segments():+ if device is not None and seg.get("device", device) != device:+ continue+ if tuple(seg.get("segment_pool_id") or ()) in pools:+ free += int(seg.get("total_size", 0)) - int(seg.get("allocated_size", 0))+ return total + max(0, free)+ except Exception: # noqa: BLE001+ return total + trackedclass _Shared:@@ -278,6 +298,7 @@ class _Shared:self.static_out: dict = {}self.static_refs: dict = {} # slot -> recordings reading its buffersself.warmed: set = set()+ self.pools: list = [] # Includes retired pools.self.pool_bytes = 0self.static_bytes = 0self.strict_device = False@@ -293,6 +314,7 @@ class _Shared:if self.pool is None:torch = _torch()self.pool = torch.cuda.graph_pool_handle()+ self.pools.append(self.pool)return self.pooldef statics_for(self, slot: tuple, live: list) -> list:@@ -337,6 +359,7 @@ class _Shared:self.static_refs.clear()self.warmed.clear()self.pool = None+ self.pools = []self.pool_bytes = 0self.static_bytes = 0@@ -587,7 +610,6 @@ class BlockGraph:dst.copy_(src)static_args, static_kwargs = _unwalk(key, iter(entry.static_in))if slot not in self.shared.warmed:- # Any guard the static buffers trip recompiles here, never inside the recording.self.compute(*static_args, **static_kwargs)self.shared.warmed.add(slot)self.stats["warmups"] += 1@@ -599,7 +621,6 @@ class BlockGraph:pool = self.shared.graph_pool()with torch.cuda.stream(stream):with _capturing():- # read after the entry, which first hands back any retired graph poolbefore = torch.cuda.memory_reserved()try:# Inside the try: a capture_begin that raised after taking the pool is abandoned too.@@ -620,9 +641,10 @@ class BlockGraph:if self.shared.pool == pool:self.shared.pool = Noneraise+ after = torch.cuda.memory_reserved()del out, flatcurrent.wait_stream(stream)- self.shared.pool_bytes += max(0, torch.cuda.memory_reserved() - before)+ self.shared.pool_bytes += max(0, after - before)except CaptureRefused:# Refused before capture_begin: nothing recorded, the pool untouched. Eager now, record on a later call.self.seen[full] = (out_spec, metas)@@ -1043,7 +1065,6 @@ class SegmentGraph:pool = self.shared.graph_pool()with torch.cuda.stream(stream):with _capturing():- # read after the entry, which first hands back any retired graph poolbefore = torch.cuda.memory_reserved()try:graph.capture_begin(pool = pool, capture_error_mode = "thread_local")@@ -1056,8 +1077,9 @@ class SegmentGraph:if self.shared.pool == pool:self.shared.pool = Noneraise+ after = torch.cuda.memory_reserved()current.wait_stream(stream)- self.shared.pool_bytes += max(0, torch.cuda.memory_reserved() - before)+ self.shared.pool_bytes += max(0, after - before)except CaptureRefused:self.seen[full] = recself.stats["segment_refusals"] = self.stats.get("segment_refusals", 0) + 1diff --git a/studio/backend/core/inference/diffusion_cuda_graph.py b/studio/backend/core/inference/diffusion_cuda_graph.pyindex 351b1673ea0..c992a179530 100644--- a/studio/backend/core/inference/diffusion_cuda_graph.py+++ b/studio/backend/core/inference/diffusion_cuda_graph.py@@ -899,12 +899,25 @@ def _prewarm_token_counts(live: list) -> tuple:return tuple(sorted(counts)[:8])+def _allocator_segments() -> list:+ torch = _torch()+ native = getattr(getattr(torch, "_C", None), "_cuda_memorySnapshot", None)+ if native is None:+ return torch.cuda.memory_snapshot()+ # Avoid borrowed-reference corruption in the tuple API: pytorch/pytorch#201043.+ try:+ snapshot = native(None)+ except TypeError:+ snapshot = native()+ return snapshot["segments"]++def _pool_bytes(graph: Any) -> int:"""Device bytes the graph's private pool holds (shared by every graph of the load), 0 if unreadable."""try:pool = tuple(graph.pool())total = 0- for seg in _torch().cuda.memory_snapshot():+ for seg in _allocator_segments():if tuple(seg.get("segment_pool_id") or ()) == pool:total += int(seg.get("total_size", 0))return total@@ -1012,8 +1025,8 @@ def live_pool_free_bytes() -> int:try:free = 0cuda = _torch().cuda- device = cuda.current_device() # the snapshot spans every card- for seg in cuda.memory_snapshot():+ device = cuda.current_device()+ for seg in _allocator_segments():if seg.get("device", device) != device:continueif tuple(seg.get("segment_pool_id") or ()) in pools:diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex 98e8cb725b6..c20d520c15d 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -1239,9 +1239,6 @@ def _metal_working_set_mib() -> tuple[Optional[int], int]:def _graph_held_cache_bytes(device: Optional[int] = None) -> int:- """Bytes inside ``memory_reserved() - memory_allocated()`` that ordinary allocations cannot reuse: the CUDA graph- pools (per-block pools whole, the whole-step graphs' shared pool's unallocated part). ``device`` None counts every- block pool and reads the whole-step pool on the current card."""held = 0try:from .diffusion_block_graph import pool_bytes as _block_graph_pool_bytesdiff --git a/studio/backend/tests/test_diffusion_block_graph.py b/studio/backend/tests/test_diffusion_block_graph.pyindex f089def3f92..55382b8868d 100644--- a/studio/backend/tests/test_diffusion_block_graph.py+++ b/studio/backend/tests/test_diffusion_block_graph.py@@ -6,6 +6,7 @@from __future__ import annotationsimport copy+import contextlibimport typesimport warnings@@ -714,6 +715,96 @@ def test_pool_bytes_count_only_the_target_card(monkeypatch):assert bg.pool_bytes(0) == 100 and bg.pool_bytes() == 1000[email protected]("legacy", [False, True])+def test_allocator_snapshot_avoids_tuple_arguments(monkeypatch, legacy):+ calls = []+ segments = [{"total_size": 123}]++ def native(*args):+ calls.append(args)+ if legacy and args:+ raise TypeError("takes no arguments")+ return {"segments": segments}++ fake = types.SimpleNamespace(_C = types.SimpleNamespace(_cuda_memorySnapshot = native))+ monkeypatch.setattr(cg, "_torch", lambda: fake)+ assert cg._allocator_segments() is segments+ assert calls == ([(None,), ()] if legacy else [(None,)])+++def test_known_pool_is_counted_when_net_reservation_is_zero(monkeypatch):+ shared = bg._Shared(0)+ shared.pools.append((7, 1))+ monkeypatch.setattr(bg, "_LIVE", {shared})+ segments = [{"device": 0, "segment_pool_id": (7, 1), "total_size": 200, "allocated_size": 80}]+ monkeypatch.setattr(torch.cuda, "memory_snapshot", lambda: segments)+ monkeypatch.setattr(+ torch._C, "_cuda_memorySnapshot", lambda *a: {"segments": segments}, raising = False+ )+ assert shared.pool_bytes == 0+ assert bg.pool_bytes(0) == 120+++def test_capture_reservation_precedes_deferred_cache_release(monkeypatch):+ _cuda()+ original = cg._capturing++ @contextlib.contextmanager+ def capturing():+ with original():+ yield+ cg.release_when_capture_idle(torch.cuda.empty_cache)++ monkeypatch.setattr(cg, "_capturing", capturing)+ block = Block().cuda()+ shared = bg._Shared(torch.cuda.current_device(), None)+ graph = bg.BlockGraph(block, block.forward, shared)+ x = torch.randn(8192, 64, device = "cuda")+ try:+ with torch.inference_mode():+ graph(x)+ idle = torch.empty(128 << 20, dtype = torch.uint8, device = "cuda")+ del idle+ graph(x)+ assert graph.stats["captures"] == 1+ assert shared.pool_bytes > 0+ assert torch.equal(graph(x), block(x))+ finally:+ graph.reset()+ shared.release()+++def test_live_segment_outputs_in_a_graph_pool_are_not_subtracted_twice(monkeypatch):+ mib = 1 << 20+ shared = bg._Shared(0)+ monkeypatch.setattr(bg, "_LIVE", {shared})+ monkeypatch.setattr(torch.cuda, "graph_pool_handle", lambda: (7, 1))+ shared.graph_pool()+ shared.pool_bytes = 500 * mib+ snapshot = [+ {+ "device": 0,+ "segment_pool_id": (7, 1),+ "total_size": 500 * mib,+ "allocated_size": 500 * mib,+ },+ {"device": 0, "segment_pool_id": (0, 0), "total_size": 100 * mib, "allocated_size": 0},+ ]+ monkeypatch.setattr(torch.cuda, "memory_snapshot", lambda: snapshot)+ monkeypatch.setattr(cg, "_allocator_segments", lambda: snapshot, raising = False)+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda *a, **k: 600 * mib)+ monkeypatch.setattr(torch.cuda, "memory_allocated", lambda *a, **k: 500 * mib)+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)+ monkeypatch.setattr(cg, "live_pool_free_bytes", lambda: 0)+ assert dm.cuda_reusable_cache_bytes(0) == 100 * mib+ snapshot[0]["allocated_size"] = 250 * mib+ monkeypatch.setattr(torch.cuda, "memory_allocated", lambda *a, **k: 250 * mib)+ assert dm.cuda_reusable_cache_bytes(0) == 100 * mib+ monkeypatch.setattr(torch.cuda, "memory_snapshot", lambda: 1 / 0)+ monkeypatch.setattr(cg, "_allocator_segments", lambda: 1 / 0, raising = False)+ assert dm.cuda_reusable_cache_bytes(0) == 0++def test_block_graph_pools_are_not_credited_as_reclaimable(monkeypatch):target = types.SimpleNamespace(device = "cuda", backend = "cuda")snap = dm.DeviceMemory("cuda", "cuda", "dedicated", 1000, 8000)diff --git a/studio/backend/tests/test_diffusion_offload_step_graphs.py b/studio/backend/tests/test_diffusion_offload_step_graphs.pyindex e83c217c65f..bd3c40dc222 100644--- a/studio/backend/tests/test_diffusion_offload_step_graphs.py+++ b/studio/backend/tests/test_diffusion_offload_step_graphs.py@@ -282,6 +282,7 @@ def test_every_live_graph_pool_stays_out_of_the_reclaimable_memory(monkeypatch):},]monkeypatch.setattr(torch.cuda, "memory_snapshot", lambda: segments)+ monkeypatch.setattr(cg, "_allocator_segments", lambda: segments, raising = False)monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)assert cg.live_pool_free_bytes() == 60