Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: CFG batch, H3 budget and stream residency fit checks do not count CUDA graph pool cache as free (#15067)

Daniel Han3049b610e98b
diff --git a/studio/backend/core/inference/diffusion_cfg_batch.py b/studio/backend/core/inference/diffusion_cfg_batch.py
index 825a6cc40e9..bd4666bc42a 100644
--- a/studio/backend/core/inference/diffusion_cfg_batch.py
+++ b/studio/backend/core/inference/diffusion_cfg_batch.py
@@ -249,14 +249,14 @@ def cfg_batch_decline(
def _usable_free_mib(device: Any) -> Optional[int]:
- """Driver free plus this process's reserved-but-unallocated cache on ``device``, less what the user's VRAM Budget keeps free (0 at the default)."""
+ """Driver free plus reusable cache (graph pools excluded) on ``device``, less the VRAM Budget reserve."""
torch = _torch()
try:
- from .diffusion_memory import _user_budget_reserve_mib
+ from .diffusion_memory import _user_budget_reserve_mib, cuda_reusable_cache_bytes
free, total = torch.cuda.mem_get_info(device)
- idle = torch.cuda.memory_reserved(device) - torch.cuda.memory_allocated(device)
- return (int(free + max(0, idle)) >> 20) - _user_budget_reserve_mib(int(total) >> 20)
+ idle = cuda_reusable_cache_bytes(device)
+ return (int(free + idle) >> 20) - _user_budget_reserve_mib(int(total) >> 20)
except Exception: # noqa: BLE001 - unreadable: the out-of-memory backstop decides
return None
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 0b575840e08..6e50ab5f17c 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -2887,8 +2887,7 @@ def _h3_residency_budget_bytes(
) -> int:
"""Device bytes the resident set may occupy for this request."""
try:
- import torch
-
+ from .diffusion_memory import cuda_reusable_cache_bytes
from .video_minimax_h3 import H3_TEXT_ENCODER_BF16_GB
from .video_minimax_h3_residency import h3_phase_need_gb
from .video_minimax_h3_te import H3_TE_STREAMED_GB
@@ -2897,8 +2896,8 @@ def _h3_residency_budget_bytes(
free = _h3_signed_free_device_bytes(device)
if free is None:
return 0
- cached = int(torch.cuda.memory_reserved()) - int(torch.cuda.memory_allocated())
- available = int(free) + max(0, cached) + int(residency.resident_bytes())
+ cached = cuda_reusable_cache_bytes(device)
+ available = int(free) + cached + int(residency.resident_bytes())
te_gb = (
H3_TE_STREAMED_GB
if te_streamed
diff --git a/studio/backend/core/inference/video_stream_residency.py b/studio/backend/core/inference/video_stream_residency.py
index 1100c7d671d..8621fe13d59 100644
--- a/studio/backend/core/inference/video_stream_residency.py
+++ b/studio/backend/core/inference/video_stream_residency.py
@@ -491,6 +491,7 @@ def fit_for_request(
from .diffusion_memory import (
_keep_groups_resident,
_user_budget_reserve_mib,
+ cuda_reusable_cache_bytes,
release_resident_groups,
)
@@ -498,7 +499,7 @@ def fit_for_request(
if dev.type != "cuda":
return None
free, total = torch.cuda.mem_get_info(dev)
- unused = int(torch.cuda.memory_reserved(dev)) - int(torch.cuda.memory_allocated(dev))
+ unused = cuda_reusable_cache_bytes(dev)
current = resident_mib(module)
# an earlier request that never reached record_request_peak (cancelled, failed) leaves nothing measured
module._unsloth_video_pending = None
diff --git a/studio/backend/tests/test_diffusion_cfg_batch.py b/studio/backend/tests/test_diffusion_cfg_batch.py
index 2cf63836b8d..db970cdf5f1 100644
--- a/studio/backend/tests/test_diffusion_cfg_batch.py
+++ b/studio/backend/tests/test_diffusion_cfg_batch.py
@@ -894,6 +894,19 @@ def test_the_verdict_store_is_keyed_on_the_card_and_skipped_off_cuda(monkeypatch
assert (tmp_path / "verdicts.json").read_text() == before
+def test_usable_free_does_not_count_graph_held_cache(monkeypatch):
+ import core.inference.diffusion_memory as dm
+
+ monkeypatch.setattr(torch.cuda, "mem_get_info", lambda device = None: (1000 * 2**20, 0))
+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda device = None: 600 * 2**20)
+ monkeypatch.setattr(torch.cuda, "memory_allocated", lambda device = None: 0)
+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)
+ monkeypatch.setattr(dm, "_user_budget_reserve_mib", lambda total_mib: 0)
+ monkeypatch.setattr(dm, "_graph_held_cache_bytes", lambda device = None: 500 * 2**20)
+ monkeypatch.setattr(cb, "_torch", lambda: torch)
+ assert cb._usable_free_mib(torch.device("cuda", 0)) == 1100
+
+
class _QuantDen(torch.nn.Module):
def __init__(self):
super().__init__()
diff --git a/studio/backend/tests/test_diffusion_memory_vram_budget.py b/studio/backend/tests/test_diffusion_memory_vram_budget.py
index 9b01daa6006..d0002207c13 100644
--- a/studio/backend/tests/test_diffusion_memory_vram_budget.py
+++ b/studio/backend/tests/test_diffusion_memory_vram_budget.py
@@ -420,6 +420,24 @@ def test_video_h3_residency_budget_keeps_the_reserve_deficit(fraction, monkeypat
) # the plain free reading still never goes negative
+def test_video_h3_residency_budget_does_not_count_graph_held_cache(monkeypatch):
+ torch = pytest.importorskip("torch")
+ import core.inference.video as vid
+ import core.inference.video_minimax_h3_residency as res
+
+ monkeypatch.setattr(vid, "_h3_signed_free_device_bytes", lambda device: 1000 * 2**20)
+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda device = None: 600 * 2**20)
+ monkeypatch.setattr(torch.cuda, "memory_allocated", lambda device = None: 0)
+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)
+ monkeypatch.setattr(dm, "_graph_held_cache_bytes", lambda device = None: 500 * 2**20)
+ monkeypatch.setattr(res, "h3_phase_need_gb", lambda *a, **k: 0.0)
+ residency = types.SimpleNamespace(resident_bytes = lambda: 0)
+ budget = vid._h3_residency_budget_bytes(
+ residency, "cuda", width = 64, height = 64, frames = 1, te_streamed = True, te_scheme = None
+ )
+ assert budget == 1100 * 2**20
+
+
def test_video_request_available_reading_charges_the_reserve(fraction, monkeypatch):
import core.inference.video as vid
from utils import hardware
@@ -590,14 +608,14 @@ def test_denoiser_hold_between_calls_keeps_the_reserve_free(fraction, monkeypatc
def test_cfg_batch_doubled_call_keeps_the_reserve_free(fraction, monkeypatch):
from core.inference import diffusion_cfg_batch as cb
- fake = types.SimpleNamespace(
- cuda = types.SimpleNamespace(
- mem_get_info = lambda device: (10 * GiB << 20, 24 * GiB << 20),
- memory_reserved = lambda device: 2 * GiB << 20,
- memory_allocated = lambda device: 1 * GiB << 20,
- )
+ torch = pytest.importorskip("torch")
+ monkeypatch.setattr(
+ torch.cuda, "mem_get_info", lambda device = None: (10 * GiB << 20, 24 * GiB << 20)
)
- monkeypatch.setattr(cb, "_torch", lambda: fake)
+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda device = None: 2 * GiB << 20)
+ monkeypatch.setattr(torch.cuda, "memory_allocated", lambda device = None: 1 * GiB << 20)
+ monkeypatch.setattr(dm, "_graph_held_cache_bytes", lambda device = None: 0)
+ monkeypatch.setattr(cb, "_torch", lambda: torch)
assert cb._usable_free_mib("cuda:0") == 11 * GiB
fraction["value"] = 0.75
assert cb._usable_free_mib("cuda:0") == 11 * GiB - 6 * GiB
diff --git a/studio/backend/tests/test_video_stream_residency.py b/studio/backend/tests/test_video_stream_residency.py
index 6d4d4adfaff..44d0e3533db 100644
--- a/studio/backend/tests/test_video_stream_residency.py
+++ b/studio/backend/tests/test_video_stream_residency.py
@@ -456,3 +456,24 @@ def test_a_per_key_drop_keeps_the_graph_key(reason):
assert per_key.placement._declined is None # the placement still records
pipe = types.SimpleNamespace(_unsloth_cuda_graphs = (per_key,))
assert vr.live_speed_optims(pipe, ("compile", "cuda_graph")) == ("compile", "cuda_graph")
+
+
+def test_fit_for_request_does_not_count_graph_held_cache(monkeypatch):
+ torch = pytest.importorskip("torch")
+ seen = {}
+
+ def room(**kw):
+ seen.update(kw)
+ raise RuntimeError("stop after the reading")
+
+ monkeypatch.setattr(torch.cuda, "mem_get_info", lambda device = None: (1000 * 2**20, 24 << 30))
+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda device = None: 600 * 2**20)
+ monkeypatch.setattr(torch.cuda, "memory_allocated", lambda device = None: 0)
+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)
+ monkeypatch.setattr(dm, "_graph_held_cache_bytes", lambda device = None: 500 * 2**20)
+ monkeypatch.setattr(vr, "resident_mib", lambda module: 0)
+ monkeypatch.setattr(vr, "room_mib", room)
+ pipe = types.SimpleNamespace(transformer = types.SimpleNamespace())
+ vr.fit_for_request(pipe, device = "cuda:0", floor_mib = 1, width = 64, height = 64, frames = 1)
+ assert seen["free_mib"] == 1000
+ assert seen["unused_cache_mib"] == 100