Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: untiled VAE fit checks do not count CUDA graph pool cache as free (#15064)

Daniel Han09e30cfa4a2f
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 6dbda38c4df..9d1d2bf465d 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -2596,7 +2596,7 @@ def _untiled_decode_shortfall_mib(vae: Any, z: Any) -> Optional[int]:
def _untiled_encode_shortfall_mib(vae: Any, x: Any) -> Optional[int]:
"""MiB short for encoding ``x`` untiled, the decode's rule (free less the VRAM Budget reserve); None off CUDA
or on unknown figures."""
- from .diffusion_memory import user_budget_reserve_bytes
+ from .diffusion_memory import cuda_reusable_cache_bytes, user_budget_reserve_bytes
from .diffusion_vae_tiling import FREE_FRACTION, _fused, _geometry, encode_mib_per_latent
try:
@@ -2609,7 +2609,7 @@ def _untiled_encode_shortfall_mib(vae: Any, x: Any) -> Optional[int]:
if per_latent is None or ratio <= 0:
return None
free, total = torch.cuda.mem_get_info(x.device)
- free += torch.cuda.memory_reserved(x.device) - torch.cuda.memory_allocated(x.device)
+ free += cuda_reusable_cache_bytes(x.device)
free -= user_budget_reserve_bytes(x.device, total)
elem = next(vae.encoder.parameters()).element_size()
# enable_fp16_vae_decode's fp32_encode upcasts the fp16 encoder and input for the encode itself
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index 030461b4bf3..bfa58e00967 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -1116,6 +1116,19 @@ def cuda_reclaimable_bytes(device: Optional[int] = None) -> Optional[int]:
return reclaimable - _graph_held_cache_bytes(pool_device)
+def cuda_reusable_cache_bytes(device: Any = None) -> int:
+ """``cuda_reclaimable_bytes`` (excludes CUDA graph pools) floored at 0 for any device form; resolved to an int
+ index because ``_graph_held_cache_bytes`` would silently skip the graph-pool count for a torch.device."""
+ index = None
+ if device is not None:
+ try:
+ import torch
+ index = device if isinstance(device, int) else torch.device(device).index
+ except Exception: # noqa: BLE001 - unparseable: read the current card
+ index = None
+ return max(0, cuda_reclaimable_bytes(index) or 0)
+
+
def reclaimable_snapshot_device_memory(target: Any) -> DeviceMemory:
"""``snapshot_device_memory`` with the caching allocator's RECLAIMABLE bytes credited back as free,
without flushing it.
diff --git a/studio/backend/core/inference/diffusion_vae_tiling.py b/studio/backend/core/inference/diffusion_vae_tiling.py
index 5bb46a097db..ade63fb6948 100644
--- a/studio/backend/core/inference/diffusion_vae_tiling.py
+++ b/studio/backend/core/inference/diffusion_vae_tiling.py
@@ -280,13 +280,13 @@ def _free_mib(
"""(free MiB after the fp32 output accumulator, decoder bytes per param / 2); None off CUDA."""
import torch
- from .diffusion_memory import user_budget_reserve_bytes
+ from .diffusion_memory import cuda_reusable_cache_bytes, user_budget_reserve_bytes
if getattr(z, "device", None) is None or z.device.type != "cuda":
return None
free, total = torch.cuda.mem_get_info(z.device)
if cached:
- free += torch.cuda.memory_reserved(z.device) - torch.cuda.memory_allocated(z.device)
+ free += cuda_reusable_cache_bytes(z.device)
free -= user_budget_reserve_bytes(z.device, total)
ratio = _geometry(vae)[0]
frames = z.shape[2] if z.dim() == 5 else 1
diff --git a/studio/backend/core/inference/video_vae_untiled.py b/studio/backend/core/inference/video_vae_untiled.py
index b35304f4a64..852986fe9bd 100644
--- a/studio/backend/core/inference/video_vae_untiled.py
+++ b/studio/backend/core/inference/video_vae_untiled.py
@@ -73,15 +73,15 @@ def _decoder_itemsize(vae: Any) -> int:
def _free_bytes(device: Any) -> Optional[int]:
- """Device free memory plus torch's unused cache, less the user's VRAM Budget reserve (raw at the default)."""
+ """Device free memory plus torch's reusable cache (not CUDA graph pools), less the user's VRAM Budget reserve (raw at the default)."""
try:
import torch
- from .diffusion_memory import user_budget_reserve_bytes
+ from .diffusion_memory import cuda_reusable_cache_bytes, user_budget_reserve_bytes
free, total = torch.cuda.mem_get_info(device)
- cached = torch.cuda.memory_reserved(device) - torch.cuda.memory_allocated(device)
- return max(0, int(free) + max(0, int(cached)) - user_budget_reserve_bytes(device, total))
+ cached = cuda_reusable_cache_bytes(device)
+ return max(0, int(free) + cached - user_budget_reserve_bytes(device, total))
except Exception: # noqa: BLE001
return None
diff --git a/studio/backend/tests/test_diffusion_release_before_tiling.py b/studio/backend/tests/test_diffusion_release_before_tiling.py
index b1d2042aae2..6a12dc91e3a 100644
--- a/studio/backend/tests/test_diffusion_release_before_tiling.py
+++ b/studio/backend/tests/test_diffusion_release_before_tiling.py
@@ -502,3 +502,44 @@ def test_the_untiled_encode_check_prices_the_fp32_encode_of_an_fp16_decode_vae(m
assert abs(diffusion._untiled_encode_shortfall_mib(vae, x) - fp16) <= 1
vae._unsloth_fp16_decode = True
assert abs(diffusion._untiled_encode_shortfall_mib(vae, x) - fp32) <= 1
+
+
+def graph_held_card(monkeypatch):
+ """1000 MiB driver free plus 600 MiB cached, 500 MiB of it a CUDA graph pool's free blocks."""
+ torch = pytest.importorskip("torch")
+ import core.inference.diffusion_memory as dm
+
+ monkeypatch.setattr(torch.cuda, "mem_get_info", lambda device = None: (1000 * 2**20, 0))
+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda device = None: 600 * 2**20)
+ monkeypatch.setattr(torch.cuda, "memory_allocated", lambda device = None: 0)
+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)
+ monkeypatch.setattr(dm, "user_budget_reserve_bytes", lambda device, total: 0)
+ monkeypatch.setattr(dm, "_graph_held_cache_bytes", lambda device = None: 500 * 2**20)
+ return types.SimpleNamespace(type = "cuda", index = 0)
+
+
+def test_the_untiled_decode_check_does_not_count_graph_held_cache(monkeypatch, graph_held_card):
+ import core.inference.diffusion_vae_tiling as tiling
+
+ monkeypatch.setattr(tiling, "_geometry", lambda vae: (8, 32, 4))
+ param = types.SimpleNamespace(element_size = lambda: 2)
+ vae = types.SimpleNamespace(decoder = types.SimpleNamespace(parameters = lambda: iter([param])))
+ z = types.SimpleNamespace(shape = (1, 4, 1, 1), dim = lambda: 4, device = graph_held_card)
+ mib, _ = tiling._free_mib(vae, z)
+ assert abs(mib - 1100) < 1
+ mib, _ = tiling._free_mib(vae, z, cached = False)
+ assert abs(mib - 1000) < 1
+
+
+def test_the_untiled_encode_check_does_not_count_graph_held_cache(monkeypatch, graph_held_card):
+ import core.inference.diffusion_vae_tiling as tiling
+
+ monkeypatch.setattr(tiling, "_geometry", lambda vae: (16, 32, 4))
+ monkeypatch.setattr(tiling, "_fused", lambda vae: False)
+ param = types.SimpleNamespace(element_size = lambda: 2)
+ vae = type("AutoencoderKLQwenImage21", (), {})()
+ vae.encoder = types.SimpleNamespace(parameters = lambda: iter([param]))
+ x = types.SimpleNamespace(shape = (1, 4, 1, 1024, 1024), dim = lambda: 5, device = graph_held_card)
+ want = 64 * 64 * 0.6 / tiling.FREE_FRACTION - 1100
+ assert abs(diffusion._untiled_encode_shortfall_mib(vae, x) - want) <= 1
diff --git a/studio/backend/tests/test_video_vae_untiled.py b/studio/backend/tests/test_video_vae_untiled.py
index 88de1cfa944..48f963437aa 100644
--- a/studio/backend/tests/test_video_vae_untiled.py
+++ b/studio/backend/tests/test_video_vae_untiled.py
@@ -182,3 +182,16 @@ def test_wrapper_survives_a_compile_fallback_replacing_the_slot(monkeypatch):
assert wrapper(Z) == ("untiled",)
assert vae.decode is wrapper
assert vae.decode(Z) == ("untiled",)
+
+
+def test_free_bytes_does_not_count_graph_held_cache(monkeypatch):
+ torch = pytest.importorskip("torch")
+ import core.inference.diffusion_memory as dm
+
+ monkeypatch.setattr(torch.cuda, "mem_get_info", lambda device = None: (1000 * 2**20, 0))
+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda device = None: 600 * 2**20)
+ monkeypatch.setattr(torch.cuda, "memory_allocated", lambda device = None: 0)
+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)
+ monkeypatch.setattr(dm, "user_budget_reserve_bytes", lambda device, total: 0)
+ monkeypatch.setattr(dm, "_graph_held_cache_bytes", lambda device = None: 500 * 2**20)
+ assert U._free_bytes(torch.device("cuda", 0)) == 1100 * 2**20