Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: untiled VAE fit checks do not count CUDA graph pool cache as free (#15064)
Daniel Han09e30cfa4a2f
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 6dbda38c4df..9d1d2bf465d 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -2596,7 +2596,7 @@ def _untiled_decode_shortfall_mib(vae: Any, z: Any) -> Optional[int]:def _untiled_encode_shortfall_mib(vae: Any, x: Any) -> Optional[int]:"""MiB short for encoding ``x`` untiled, the decode's rule (free less the VRAM Budget reserve); None off CUDAor on unknown figures."""- from .diffusion_memory import user_budget_reserve_bytes+ from .diffusion_memory import cuda_reusable_cache_bytes, user_budget_reserve_bytesfrom .diffusion_vae_tiling import FREE_FRACTION, _fused, _geometry, encode_mib_per_latenttry:@@ -2609,7 +2609,7 @@ def _untiled_encode_shortfall_mib(vae: Any, x: Any) -> Optional[int]:if per_latent is None or ratio <= 0:return Nonefree, total = torch.cuda.mem_get_info(x.device)- free += torch.cuda.memory_reserved(x.device) - torch.cuda.memory_allocated(x.device)+ free += cuda_reusable_cache_bytes(x.device)free -= user_budget_reserve_bytes(x.device, total)elem = next(vae.encoder.parameters()).element_size()# enable_fp16_vae_decode's fp32_encode upcasts the fp16 encoder and input for the encode itselfdiff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex 030461b4bf3..bfa58e00967 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -1116,6 +1116,19 @@ def cuda_reclaimable_bytes(device: Optional[int] = None) -> Optional[int]:return reclaimable - _graph_held_cache_bytes(pool_device)+def cuda_reusable_cache_bytes(device: Any = None) -> int:+ """``cuda_reclaimable_bytes`` (excludes CUDA graph pools) floored at 0 for any device form; resolved to an int+ index because ``_graph_held_cache_bytes`` would silently skip the graph-pool count for a torch.device."""+ index = None+ if device is not None:+ try:+ import torch+ index = device if isinstance(device, int) else torch.device(device).index+ except Exception: # noqa: BLE001 - unparseable: read the current card+ index = None+ return max(0, cuda_reclaimable_bytes(index) or 0)++def reclaimable_snapshot_device_memory(target: Any) -> DeviceMemory:"""``snapshot_device_memory`` with the caching allocator's RECLAIMABLE bytes credited back as free,without flushing it.diff --git a/studio/backend/core/inference/diffusion_vae_tiling.py b/studio/backend/core/inference/diffusion_vae_tiling.pyindex 5bb46a097db..ade63fb6948 100644--- a/studio/backend/core/inference/diffusion_vae_tiling.py+++ b/studio/backend/core/inference/diffusion_vae_tiling.py@@ -280,13 +280,13 @@ def _free_mib("""(free MiB after the fp32 output accumulator, decoder bytes per param / 2); None off CUDA."""import torch- from .diffusion_memory import user_budget_reserve_bytes+ from .diffusion_memory import cuda_reusable_cache_bytes, user_budget_reserve_bytesif getattr(z, "device", None) is None or z.device.type != "cuda":return Nonefree, total = torch.cuda.mem_get_info(z.device)if cached:- free += torch.cuda.memory_reserved(z.device) - torch.cuda.memory_allocated(z.device)+ free += cuda_reusable_cache_bytes(z.device)free -= user_budget_reserve_bytes(z.device, total)ratio = _geometry(vae)[0]frames = z.shape[2] if z.dim() == 5 else 1diff --git a/studio/backend/core/inference/video_vae_untiled.py b/studio/backend/core/inference/video_vae_untiled.pyindex b35304f4a64..852986fe9bd 100644--- a/studio/backend/core/inference/video_vae_untiled.py+++ b/studio/backend/core/inference/video_vae_untiled.py@@ -73,15 +73,15 @@ def _decoder_itemsize(vae: Any) -> int:def _free_bytes(device: Any) -> Optional[int]:- """Device free memory plus torch's unused cache, less the user's VRAM Budget reserve (raw at the default)."""+ """Device free memory plus torch's reusable cache (not CUDA graph pools), less the user's VRAM Budget reserve (raw at the default)."""try:import torch- from .diffusion_memory import user_budget_reserve_bytes+ from .diffusion_memory import cuda_reusable_cache_bytes, user_budget_reserve_bytesfree, total = torch.cuda.mem_get_info(device)- cached = torch.cuda.memory_reserved(device) - torch.cuda.memory_allocated(device)- return max(0, int(free) + max(0, int(cached)) - user_budget_reserve_bytes(device, total))+ cached = cuda_reusable_cache_bytes(device)+ return max(0, int(free) + cached - user_budget_reserve_bytes(device, total))except Exception: # noqa: BLE001return Nonediff --git a/studio/backend/tests/test_diffusion_release_before_tiling.py b/studio/backend/tests/test_diffusion_release_before_tiling.pyindex b1d2042aae2..6a12dc91e3a 100644--- a/studio/backend/tests/test_diffusion_release_before_tiling.py+++ b/studio/backend/tests/test_diffusion_release_before_tiling.py@@ -502,3 +502,44 @@ def test_the_untiled_encode_check_prices_the_fp32_encode_of_an_fp16_decode_vae(massert abs(diffusion._untiled_encode_shortfall_mib(vae, x) - fp16) <= 1vae._unsloth_fp16_decode = Trueassert abs(diffusion._untiled_encode_shortfall_mib(vae, x) - fp32) <= 1+++def graph_held_card(monkeypatch):+ """1000 MiB driver free plus 600 MiB cached, 500 MiB of it a CUDA graph pool's free blocks."""+ torch = pytest.importorskip("torch")+ import core.inference.diffusion_memory as dm++ monkeypatch.setattr(torch.cuda, "mem_get_info", lambda device = None: (1000 * 2**20, 0))+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda device = None: 600 * 2**20)+ monkeypatch.setattr(torch.cuda, "memory_allocated", lambda device = None: 0)+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)+ monkeypatch.setattr(dm, "user_budget_reserve_bytes", lambda device, total: 0)+ monkeypatch.setattr(dm, "_graph_held_cache_bytes", lambda device = None: 500 * 2**20)+ return types.SimpleNamespace(type = "cuda", index = 0)+++def test_the_untiled_decode_check_does_not_count_graph_held_cache(monkeypatch, graph_held_card):+ import core.inference.diffusion_vae_tiling as tiling++ monkeypatch.setattr(tiling, "_geometry", lambda vae: (8, 32, 4))+ param = types.SimpleNamespace(element_size = lambda: 2)+ vae = types.SimpleNamespace(decoder = types.SimpleNamespace(parameters = lambda: iter([param])))+ z = types.SimpleNamespace(shape = (1, 4, 1, 1), dim = lambda: 4, device = graph_held_card)+ mib, _ = tiling._free_mib(vae, z)+ assert abs(mib - 1100) < 1+ mib, _ = tiling._free_mib(vae, z, cached = False)+ assert abs(mib - 1000) < 1+++def test_the_untiled_encode_check_does_not_count_graph_held_cache(monkeypatch, graph_held_card):+ import core.inference.diffusion_vae_tiling as tiling++ monkeypatch.setattr(tiling, "_geometry", lambda vae: (16, 32, 4))+ monkeypatch.setattr(tiling, "_fused", lambda vae: False)+ param = types.SimpleNamespace(element_size = lambda: 2)+ vae = type("AutoencoderKLQwenImage21", (), {})()+ vae.encoder = types.SimpleNamespace(parameters = lambda: iter([param]))+ x = types.SimpleNamespace(shape = (1, 4, 1, 1024, 1024), dim = lambda: 5, device = graph_held_card)+ want = 64 * 64 * 0.6 / tiling.FREE_FRACTION - 1100+ assert abs(diffusion._untiled_encode_shortfall_mib(vae, x) - want) <= 1diff --git a/studio/backend/tests/test_video_vae_untiled.py b/studio/backend/tests/test_video_vae_untiled.pyindex 88de1cfa944..48f963437aa 100644--- a/studio/backend/tests/test_video_vae_untiled.py+++ b/studio/backend/tests/test_video_vae_untiled.py@@ -182,3 +182,16 @@ def test_wrapper_survives_a_compile_fallback_replacing_the_slot(monkeypatch):assert wrapper(Z) == ("untiled",)assert vae.decode is wrapperassert vae.decode(Z) == ("untiled",)+++def test_free_bytes_does_not_count_graph_held_cache(monkeypatch):+ torch = pytest.importorskip("torch")+ import core.inference.diffusion_memory as dm++ monkeypatch.setattr(torch.cuda, "mem_get_info", lambda device = None: (1000 * 2**20, 0))+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda device = None: 600 * 2**20)+ monkeypatch.setattr(torch.cuda, "memory_allocated", lambda device = None: 0)+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)+ monkeypatch.setattr(dm, "user_budget_reserve_bytes", lambda device, total: 0)+ monkeypatch.setattr(dm, "_graph_held_cache_bytes", lambda device = None: 500 * 2**20)+ assert U._free_bytes(torch.device("cuda", 0)) == 1100 * 2**20