Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio video: keep per-request residency fitting after the stream step graph is dropped (Wan2.2-TI2V-5B at 8 / 12 GB) (#14937)

Daniel Han598ea6ab580a
diff --git a/studio/backend/core/inference/diffusion_block_graph.py b/studio/backend/core/inference/diffusion_block_graph.py
index 02e006e59fd..6932504c1da 100644
--- a/studio/backend/core/inference/diffusion_block_graph.py
+++ b/studio/backend/core/inference/diffusion_block_graph.py
@@ -655,8 +655,9 @@ class BlockGraph:
from .diffusion_cuda_graph import capture_device, note_capture
self._unsloth_capture_device = capture_device()
- note_capture(self._unsloth_capture_device)
+ # counted live before the generation moves, so a reader seeing an unchanged generation never misses it
_LIVE_BLOCKS.add(self)
+ note_capture(self._unsloth_capture_device)
while len(self.cache) > self.max_graphs:
_drop_entries(self, [self.cache.popitem(last = False)[1]], evicting = True)
self.stats["evictions"] += 1
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 248d2e7e9ed..2395f594b8f 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -11936,7 +11936,9 @@ class VideoBackend:
dtype = getattr(state, "dtype", None),
attention_backend = getattr(state, "attention_backend", None),
speed_mode = getattr(state, "speed_mode", None),
- speed_optims = tuple(getattr(state, "speed_optims", ()) or ()),
+ speed_optims = video_stream_residency.live_speed_optims(
+ state.pipe, getattr(state, "speed_optims", ())
+ ),
transformer_cache = request_cache_mode,
vae_tiling = getattr(state, "vae_tiling", None),
memory_mode = getattr(state, "memory_mode", None),
diff --git a/studio/backend/core/inference/video_stream_residency.py b/studio/backend/core/inference/video_stream_residency.py
index 1edbace75f2..b6fc6e4188e 100644
--- a/studio/backend/core/inference/video_stream_residency.py
+++ b/studio/backend/core/inference/video_stream_residency.py
@@ -283,10 +283,15 @@ def _open_window(module: Any, dev: Any) -> Any:
if stale is not None:
close_window(stale[0], stale[1])
module._unsloth_video_window = (dev, open_window(dev))
+ before = _capture_generation(dev)
graphs = _live_graphs(dev)
generation = _capture_generation(dev)
accounted = getattr(module, "_unsloth_video_graphs", None)
- if accounted is None or graphs < accounted:
+ # a capture counts its graph before its generation, so an unchanged generation around the count makes 0 consistent
+ if graphs == 0 and generation == before:
+ module._unsloth_video_graphs = accounted = 0
+ module._unsloth_video_graph_gen = generation
+ elif accounted is None or graphs < accounted:
# the load's own graphs (every request starts above them), or fewer after a reset freed some
first = accounted is None
module._unsloth_video_graphs = accounted = graphs
@@ -345,6 +350,38 @@ def _failed_captures() -> int:
return 0
+def _graphs_off_for_load(handles: tuple) -> bool:
+ """True when every whole-step graph is off for good (declined or poisoned, no per-block fallback); per-call refusals keep the key."""
+ if not handles:
+ return False
+ for h in handles:
+ if (
+ getattr(h, "fallback", None) is not None
+ or getattr(h, "fallback_handle", None) is not None
+ ):
+ return False
+ # load-wide only: a per-key drop (probe OOM, unpaid replay) leaves other shapes free to capture
+ placement = getattr(h, "placement", None)
+ declined = placement is not None and getattr(placement, "_declined", None) is not None
+ dropped = declined and not getattr(h, "cache", None)
+ if not (dropped or getattr(h, "poisoned", False) is True):
+ return False
+ return True
+
+
+def live_speed_optims(pipe: Any, speed_optims: Any) -> tuple:
+ """``speed_optims`` minus ``cuda_graph`` once this load's graphs are off for good, so the key matches what runs."""
+ optims = tuple(speed_optims or ())
+ if "cuda_graph" not in optims:
+ return optims
+ try:
+ if _graphs_off_for_load(tuple(getattr(pipe, "_unsloth_cuda_graphs", ()) or ())):
+ return tuple(o for o in optims if o != "cuda_graph")
+ return optims
+ except Exception: # noqa: BLE001 - unreadable: keep the load's record
+ return optims
+
+
def _keyed_graphs_live(pipe: Any, key: Any) -> bool:
"""False when ``key`` names CUDA graphs that ``live_status`` reports dropped or never engaged for this load."""
try:
diff --git a/studio/backend/tests/test_video_stream_residency.py b/studio/backend/tests/test_video_stream_residency.py
index 1832e15f708..77a6af1785c 100644
--- a/studio/backend/tests/test_video_stream_residency.py
+++ b/studio/backend/tests/test_video_stream_residency.py
@@ -285,3 +285,159 @@ def test_measured_peak_widens_the_next_request(monkeypatch):
monkeypatch.delenv(vr.VIDEO_DIT_RESIDENT_MEASURED_ENV)
vr.fit_for_request(pipe, device = "cuda", floor_mib = 1, width = 64, height = 64, frames = 2)
assert _placed(net) == ["cuda"] + ["cpu"] * 5
+
+
+def _graph_handle(declined = None, **state):
+ torch = pytest.importorskip("torch")
+ import core.inference.diffusion_cuda_graph as cg
+
+ module = torch.nn.Linear(2, 2)
+ module.forward = module.forward # a group offload forward in the instance slot
+ placement = cg.OffloadPlacement(module, "group")
+ if declined is not None:
+ placement.decline(declined) # what the streamed speed check does for the whole load
+ handle = cg.GraphedForward(module, placement = placement)
+ for name, value in state.items():
+ setattr(handle, name, value)
+ return handle
+
+
+_DROP = "replay measured within 1% of the eager step on this device (102.1 vs 102.7 ms)"
+
+
+def test_a_dropped_graph_keys_the_request_on_the_eager_step():
+ optims = ("compile", "cuda_graph")
+ # the speed check dropped the only graph for the load: steps run eager from here on
+ dropped = _graph_handle(
+ declined = _DROP, _slower = _DROP, capture_error = {"type": "Refused", "msg": _DROP}
+ )
+ pipe = types.SimpleNamespace(_unsloth_cuda_graphs = (dropped,))
+ assert vr.live_speed_optims(pipe, optims) == ("compile",)
+ key = vr.calibration_key("wan2.2-ti2v-5b", speed_optims = vr.live_speed_optims(pipe, optims))
+ assert vr._keyed_graphs_live(pipe, key)
+ poisoned = _graph_handle(poisoned = True)
+ assert vr.live_speed_optims(
+ types.SimpleNamespace(_unsloth_cuda_graphs = (poisoned,)), optims
+ ) == ("compile",)
+ # not armed yet, or no cuda_graph to strip: the load's record stands
+ assert vr.live_speed_optims(types.SimpleNamespace(), optims) == optims
+ assert vr.live_speed_optims(pipe, ("compile",)) == ("compile",)
+
+
[email protected]("attr", ["fallback", "fallback_handle"])
+def test_a_dropped_step_graph_with_per_block_graphs_keeps_the_graph_key(attr):
+ # the drop arms (or engaged) the per-block fallback, whose graphs run the next requests
+ dropped = _graph_handle(
+ declined = _DROP,
+ _slower = _DROP,
+ capture_error = {"type": "Refused", "msg": _DROP},
+ **{attr: object()},
+ )
+ pipe = types.SimpleNamespace(_unsloth_cuda_graphs = (dropped,))
+ assert vr.live_speed_optims(pipe, ("compile", "cuda_graph")) == ("compile", "cuda_graph")
+
+
+def test_a_request_specific_eager_call_keeps_the_graph_key():
+ import core.inference.diffusion_cuda_graph as cg
+
+ # every call so far refused one argument: still armed, a later request can capture and replay
+ refused = _graph_handle()
+ refused.stats.update(eager_calls = 16, refused_object = 16)
+ assert cg.never_engaged((refused,)) is not None
+ pipe = types.SimpleNamespace(_unsloth_cuda_graphs = (refused,))
+ assert vr.live_speed_optims(pipe, ("compile", "cuda_graph")) == ("compile", "cuda_graph")
+ # a placement refusal (Refused, but no speed verdict) is per call too
+ placed = _graph_handle(capture_error = {"type": "Refused", "msg": "a placement refusal"})
+ pipe = types.SimpleNamespace(_unsloth_cuda_graphs = (placed,))
+ assert vr.live_speed_optims(pipe, ("compile", "cuda_graph")) == ("compile", "cuda_graph")
+
+
+def test_a_capture_racing_the_zero_count_keeps_the_request_dirty(monkeypatch):
+ torch = pytest.importorskip("torch")
+ import core.inference.diffusion_measured_peaks as mp
+
+ gen = [3]
+
+ def count(dev):
+ gen[0] += 1 # another pipeline captures on the card right after the count was read
+ return 0
+
+ monkeypatch.setattr(vr, "_live_graphs", count)
+ monkeypatch.setattr(vr, "_capture_generation", lambda dev: gen[0])
+ monkeypatch.setattr(mp, "open_window", lambda dev: object())
+ monkeypatch.setattr(mp, "close_window", lambda dev, token: True)
+ module = types.SimpleNamespace(_unsloth_video_graphs = 0, _unsloth_video_graph_gen = 3)
+ vr._open_window(module, torch.device("cuda", 0))
+ assert not module._unsloth_video_graphs_clean
+ assert module._unsloth_video_graph_gen == 3
+
+
+def test_video_request_key_names_the_live_speed_optims():
+ src = (Path(__file__).resolve().parents[1] / "core" / "inference" / "video.py").read_text(
+ encoding = "utf-8"
+ )
+ key = src[src.index("calibration = video_stream_residency.calibration_key(") :]
+ key = key[: key.index("video_stream_residency.arm_request(")]
+ assert "speed_optims = video_stream_residency.live_speed_optims(" in key
+ assert 'speed_optims = tuple(getattr(state, "speed_optims"' not in key
+
+
+def test_a_request_after_a_dropped_graph_is_recorded_under_its_key(monkeypatch):
+ torch = pytest.importorskip("torch")
+ import core.inference.diffusion_measured_peaks as mp
+
+ monkeypatch.setattr(vr, "_PEAKS", {})
+ live, gen = [0], [0]
+ monkeypatch.setattr(vr, "_live_graphs", lambda dev: live[0])
+ monkeypatch.setattr(vr, "_capture_generation", lambda dev: gen[0])
+ monkeypatch.setattr(vr, "_store_key", lambda key: None)
+ monkeypatch.setattr(mp, "open_window", lambda dev: object())
+ monkeypatch.setattr(mp, "close_window", lambda dev, token: True)
+ monkeypatch.setattr(dm, "stream_window_pad_mib", lambda pipe: 0)
+ monkeypatch.setattr(torch.cuda, "max_memory_allocated", lambda *a: 3000 << 20)
+ key = vr.calibration_key("wan2.2-ti2v-5b", speed_optims = ("compile",))
+ module = types.SimpleNamespace(_unsloth_video_calibration_key = key)
+ pipe = types.SimpleNamespace(transformer = module)
+ dev = torch.device("cuda", 0)
+ vr._open_window(module, dev)
+ assert module._unsloth_video_graphs_clean
+ live[0], gen[0] = 0, 1 # it captured one graph, which the speed check then dropped (pool freed)
+ vr._open_window(module, dev)
+ assert module._unsloth_video_graphs_clean
+ module._unsloth_video_pending = (100, 1000 << 20, dev)
+ assert vr.record_request_peak(pipe) == 2000
+ assert vr.measured_extra_mib(key, 100) == 2000
+ # a graph alive from an unrecorded capture keeps the next request dirty
+ live[0], gen[0] = 1, 2
+ vr._open_window(module, dev)
+ assert not module._unsloth_video_graphs_clean
+
+
+def test_every_capture_is_counted_live_before_its_generation_moves():
+ root = Path(__file__).resolve().parents[1] / "core" / "inference"
+ block = (root / "diffusion_block_graph.py").read_text(encoding = "utf-8")
+ note = "note_capture(self._unsloth_capture_device)"
+ for at in [i for i in range(len(block)) if block.startswith(note, i)]:
+ body = block[block.rindex("\n def ", 0, at) : at]
+ assert "self.cache[full] = entry" in body
+ assert "self._unsloth_capture_device = capture_device()" in body
+ if "_LIVE_BLOCKS.add(self)" in block[at : block.index("\n def ", at)]:
+ raise AssertionError("a block joins the live set only after its generation moved")
+ step = (root / "diffusion_cuda_graph.py").read_text(encoding = "utf-8")
+ at = step.index(" note_capture(self._unsloth_capture_device)")
+ assert step.rindex("self.cache[key] = entry", 0, at) < at
+
+
+ "reason",
+ [
+ "no memory for the speed probe's input copies",
+ "replay measured slower than the eager step on this device (110.0 vs 100.0 ms)",
+ ],
+)
+def test_a_per_key_drop_keeps_the_graph_key(reason):
+ # a probe OOM, or an unpaid replay on a resident placement, drops one key only: another shape can still capture
+ per_key = _graph_handle(_slower = reason, capture_error = {"type": "Refused", "msg": reason})
+ assert per_key.placement._declined is None # the placement still records
+ pipe = types.SimpleNamespace(_unsloth_cuda_graphs = (per_key,))
+ assert vr.live_speed_optims(pipe, ("compile", "cuda_graph")) == ("compile", "cuda_graph")