Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: CFG batch, H3 budget and stream residency fit checks do not count CUDA graph pool cache as free (#15067)
Daniel Han3049b610e98b
diff --git a/studio/backend/core/inference/diffusion_cfg_batch.py b/studio/backend/core/inference/diffusion_cfg_batch.pyindex 825a6cc40e9..bd4666bc42a 100644--- a/studio/backend/core/inference/diffusion_cfg_batch.py+++ b/studio/backend/core/inference/diffusion_cfg_batch.py@@ -249,14 +249,14 @@ def cfg_batch_decline(def _usable_free_mib(device: Any) -> Optional[int]:- """Driver free plus this process's reserved-but-unallocated cache on ``device``, less what the user's VRAM Budget keeps free (0 at the default)."""+ """Driver free plus reusable cache (graph pools excluded) on ``device``, less the VRAM Budget reserve."""torch = _torch()try:- from .diffusion_memory import _user_budget_reserve_mib+ from .diffusion_memory import _user_budget_reserve_mib, cuda_reusable_cache_bytesfree, total = torch.cuda.mem_get_info(device)- idle = torch.cuda.memory_reserved(device) - torch.cuda.memory_allocated(device)- return (int(free + max(0, idle)) >> 20) - _user_budget_reserve_mib(int(total) >> 20)+ idle = cuda_reusable_cache_bytes(device)+ return (int(free + idle) >> 20) - _user_budget_reserve_mib(int(total) >> 20)except Exception: # noqa: BLE001 - unreadable: the out-of-memory backstop decidesreturn Nonediff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 0b575840e08..6e50ab5f17c 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -2887,8 +2887,7 @@ def _h3_residency_budget_bytes() -> int:"""Device bytes the resident set may occupy for this request."""try:- import torch-+ from .diffusion_memory import cuda_reusable_cache_bytesfrom .video_minimax_h3 import H3_TEXT_ENCODER_BF16_GBfrom .video_minimax_h3_residency import h3_phase_need_gbfrom .video_minimax_h3_te import H3_TE_STREAMED_GB@@ -2897,8 +2896,8 @@ def _h3_residency_budget_bytes(free = _h3_signed_free_device_bytes(device)if free is None:return 0- cached = int(torch.cuda.memory_reserved()) - int(torch.cuda.memory_allocated())- available = int(free) + max(0, cached) + int(residency.resident_bytes())+ cached = cuda_reusable_cache_bytes(device)+ available = int(free) + cached + int(residency.resident_bytes())te_gb = (H3_TE_STREAMED_GBif te_streameddiff --git a/studio/backend/core/inference/video_stream_residency.py b/studio/backend/core/inference/video_stream_residency.pyindex 1100c7d671d..8621fe13d59 100644--- a/studio/backend/core/inference/video_stream_residency.py+++ b/studio/backend/core/inference/video_stream_residency.py@@ -491,6 +491,7 @@ def fit_for_request(from .diffusion_memory import (_keep_groups_resident,_user_budget_reserve_mib,+ cuda_reusable_cache_bytes,release_resident_groups,)@@ -498,7 +499,7 @@ def fit_for_request(if dev.type != "cuda":return Nonefree, total = torch.cuda.mem_get_info(dev)- unused = int(torch.cuda.memory_reserved(dev)) - int(torch.cuda.memory_allocated(dev))+ unused = cuda_reusable_cache_bytes(dev)current = resident_mib(module)# an earlier request that never reached record_request_peak (cancelled, failed) leaves nothing measuredmodule._unsloth_video_pending = Nonediff --git a/studio/backend/tests/test_diffusion_cfg_batch.py b/studio/backend/tests/test_diffusion_cfg_batch.pyindex 2cf63836b8d..db970cdf5f1 100644--- a/studio/backend/tests/test_diffusion_cfg_batch.py+++ b/studio/backend/tests/test_diffusion_cfg_batch.py@@ -894,6 +894,19 @@ def test_the_verdict_store_is_keyed_on_the_card_and_skipped_off_cuda(monkeypatchassert (tmp_path / "verdicts.json").read_text() == before+def test_usable_free_does_not_count_graph_held_cache(monkeypatch):+ import core.inference.diffusion_memory as dm++ monkeypatch.setattr(torch.cuda, "mem_get_info", lambda device = None: (1000 * 2**20, 0))+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda device = None: 600 * 2**20)+ monkeypatch.setattr(torch.cuda, "memory_allocated", lambda device = None: 0)+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)+ monkeypatch.setattr(dm, "_user_budget_reserve_mib", lambda total_mib: 0)+ monkeypatch.setattr(dm, "_graph_held_cache_bytes", lambda device = None: 500 * 2**20)+ monkeypatch.setattr(cb, "_torch", lambda: torch)+ assert cb._usable_free_mib(torch.device("cuda", 0)) == 1100++class _QuantDen(torch.nn.Module):def __init__(self):super().__init__()diff --git a/studio/backend/tests/test_diffusion_memory_vram_budget.py b/studio/backend/tests/test_diffusion_memory_vram_budget.pyindex 9b01daa6006..d0002207c13 100644--- a/studio/backend/tests/test_diffusion_memory_vram_budget.py+++ b/studio/backend/tests/test_diffusion_memory_vram_budget.py@@ -420,6 +420,24 @@ def test_video_h3_residency_budget_keeps_the_reserve_deficit(fraction, monkeypat) # the plain free reading still never goes negative+def test_video_h3_residency_budget_does_not_count_graph_held_cache(monkeypatch):+ torch = pytest.importorskip("torch")+ import core.inference.video as vid+ import core.inference.video_minimax_h3_residency as res++ monkeypatch.setattr(vid, "_h3_signed_free_device_bytes", lambda device: 1000 * 2**20)+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda device = None: 600 * 2**20)+ monkeypatch.setattr(torch.cuda, "memory_allocated", lambda device = None: 0)+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)+ monkeypatch.setattr(dm, "_graph_held_cache_bytes", lambda device = None: 500 * 2**20)+ monkeypatch.setattr(res, "h3_phase_need_gb", lambda *a, **k: 0.0)+ residency = types.SimpleNamespace(resident_bytes = lambda: 0)+ budget = vid._h3_residency_budget_bytes(+ residency, "cuda", width = 64, height = 64, frames = 1, te_streamed = True, te_scheme = None+ )+ assert budget == 1100 * 2**20++def test_video_request_available_reading_charges_the_reserve(fraction, monkeypatch):import core.inference.video as vidfrom utils import hardware@@ -590,14 +608,14 @@ def test_denoiser_hold_between_calls_keeps_the_reserve_free(fraction, monkeypatcdef test_cfg_batch_doubled_call_keeps_the_reserve_free(fraction, monkeypatch):from core.inference import diffusion_cfg_batch as cb- fake = types.SimpleNamespace(- cuda = types.SimpleNamespace(- mem_get_info = lambda device: (10 * GiB << 20, 24 * GiB << 20),- memory_reserved = lambda device: 2 * GiB << 20,- memory_allocated = lambda device: 1 * GiB << 20,- )+ torch = pytest.importorskip("torch")+ monkeypatch.setattr(+ torch.cuda, "mem_get_info", lambda device = None: (10 * GiB << 20, 24 * GiB << 20))- monkeypatch.setattr(cb, "_torch", lambda: fake)+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda device = None: 2 * GiB << 20)+ monkeypatch.setattr(torch.cuda, "memory_allocated", lambda device = None: 1 * GiB << 20)+ monkeypatch.setattr(dm, "_graph_held_cache_bytes", lambda device = None: 0)+ monkeypatch.setattr(cb, "_torch", lambda: torch)assert cb._usable_free_mib("cuda:0") == 11 * GiBfraction["value"] = 0.75assert cb._usable_free_mib("cuda:0") == 11 * GiB - 6 * GiBdiff --git a/studio/backend/tests/test_video_stream_residency.py b/studio/backend/tests/test_video_stream_residency.pyindex 6d4d4adfaff..44d0e3533db 100644--- a/studio/backend/tests/test_video_stream_residency.py+++ b/studio/backend/tests/test_video_stream_residency.py@@ -456,3 +456,24 @@ def test_a_per_key_drop_keeps_the_graph_key(reason):assert per_key.placement._declined is None # the placement still recordspipe = types.SimpleNamespace(_unsloth_cuda_graphs = (per_key,))assert vr.live_speed_optims(pipe, ("compile", "cuda_graph")) == ("compile", "cuda_graph")+++def test_fit_for_request_does_not_count_graph_held_cache(monkeypatch):+ torch = pytest.importorskip("torch")+ seen = {}++ def room(**kw):+ seen.update(kw)+ raise RuntimeError("stop after the reading")++ monkeypatch.setattr(torch.cuda, "mem_get_info", lambda device = None: (1000 * 2**20, 24 << 30))+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda device = None: 600 * 2**20)+ monkeypatch.setattr(torch.cuda, "memory_allocated", lambda device = None: 0)+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)+ monkeypatch.setattr(dm, "_graph_held_cache_bytes", lambda device = None: 500 * 2**20)+ monkeypatch.setattr(vr, "resident_mib", lambda module: 0)+ monkeypatch.setattr(vr, "room_mib", room)+ pipe = types.SimpleNamespace(transformer = types.SimpleNamespace())+ vr.fit_for_request(pipe, device = "cuda:0", floor_mib = 1, width = 64, height = 64, frames = 1)+ assert seen["free_mib"] == 1000+ assert seen["unused_cache_mib"] == 100