Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep Wan2.2-5B's leading blocks resident under group offload (#12691)

Daniel Han6f4d9974dcea
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index f9b1a9cf4a0..d463fe592bc 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -8649,6 +8649,24 @@ class VideoBackend:
if ordinal is not None
else torch.device(state.device)
)
+ from . import video_stream_residency
+
+ if video_stream_residency.applies(
+ fam.name,
+ is_moe = bool(getattr(fam, "is_moe", False)),
+ offload_policy = state.offload_policy,
+ device = state.device,
+ ):
+ # Resident groups are allocated, so the reserved term below still counts them as available.
+ video_stream_residency.fit_for_request(
+ state.pipe,
+ device = device_obj,
+ floor_mib = state.vram_floor_mib,
+ width = width,
+ height = height,
+ frames = frames,
+ logger = logger,
+ )
free_bytes, _ = trusted_mem_get_info(device_obj, module = torch.cuda)
reserved_bytes = (
torch.cuda.memory_reserved(device_obj)
@@ -9053,6 +9071,11 @@ class VideoBackend:
# A cancel during the blocking export/mux must still discard the clip; re-check before it is persisted.
if cancel.is_set():
raise RuntimeError(VIDEO_CANCELLED_MSG)
+ if len(video_frames) and not fam.modular_workflow:
+ from . import video_stream_residency
+
+ # after decode and export: this peak sizes the next request of its size
+ video_stream_residency.record_request_peak(pipe, logger = logger)
duration_s = len(video_frames) / float(out_fps) if out_fps else 0.0
self._gen = {"active": False}
# Deregister under cancel_generate's own lock before the trim: it blocks for a few
diff --git a/studio/backend/core/inference/video_stream_residency.py b/studio/backend/core/inference/video_stream_residency.py
new file mode 100644
index 00000000000..d79f68476b4
--- /dev/null
+++ b/studio/backend/core/inference/video_stream_residency.py
@@ -0,0 +1,280 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Keep the top-level group and a block prefix of a group-offloaded Wan denoiser resident, re-fitted per request.
+
+Room = free + unused cache + already resident - need. Need is the streamed floor + base overhead + the runtime estimate,
+or, for a request no larger (pixels x frames) than a completed one, that request's measured peak x 1.15 + 1 GiB; the
+estimate stays the lower bound. Resident groups keep their host copy, so streaming them again is a pointer swap.
+Kill switches: UNSLOTH_VIDEO_DIT_RESIDENT=0, UNSLOTH_VIDEO_DIT_RESIDENT_BLOCKS=0 (top-level group only),
+UNSLOTH_VIDEO_DIT_RESIDENT_MEASURED=0 (estimate only).
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any, Optional
+
+VIDEO_DIT_RESIDENT_ENV = "UNSLOTH_VIDEO_DIT_RESIDENT"
+VIDEO_DIT_RESIDENT_BLOCKS_ENV = "UNSLOTH_VIDEO_DIT_RESIDENT_BLOCKS"
+VIDEO_DIT_RESIDENT_MEASURED_ENV = "UNSLOTH_VIDEO_DIT_RESIDENT_MEASURED"
+# A measured peak is scaled by this and a fixed slack is kept on top (allocator fragmentation, cuDNN workspaces).
+MEASURED_PEAK_MARGIN = 1.15
+MEASURED_SLACK_MIB = 1024
+
+_COUNTS: dict[str, int] = {
+ "fits": 0,
+ "measured_fits": 0,
+ "recorded": 0,
+ "promoted_mib": 0,
+ "released_mib": 0,
+ "resident_mib": 0,
+ "groups": 0,
+}
+
+
+def counts() -> dict[str, int]:
+ return dict(_COUNTS)
+
+
+def _off(name: str) -> bool:
+ return (os.environ.get(name) or "").strip().lower() in ("0", "off", "false", "no")
+
+
+def residency_enabled() -> bool:
+ return not _off(VIDEO_DIT_RESIDENT_ENV)
+
+
+def blocks_enabled() -> bool:
+ return not _off(VIDEO_DIT_RESIDENT_BLOCKS_ENV)
+
+
+def measured_enabled() -> bool:
+ return not _off(VIDEO_DIT_RESIDENT_MEASURED_ENV)
+
+
+def applies(
+ family_name: Optional[str],
+ *,
+ is_moe: bool,
+ offload_policy: Optional[str],
+ device: Any,
+ modular: bool = False,
+) -> bool:
+ return (
+ residency_enabled()
+ and not modular
+ and not is_moe
+ and str(family_name or "").startswith("wan")
+ and offload_policy == "group"
+ and str(device or "").startswith("cuda")
+ )
+
+
+def resident_bytes(module: Any) -> int:
+ from .diffusion_memory import _offload_groups
+
+ total = 0
+ for group in _offload_groups(module) or ():
+ if getattr(group, "_unsloth_resident", False):
+ total += int(getattr(group, "_unsloth_resident_bytes", 0) or 0)
+ return total
+
+
+def resident_mib(module: Any) -> int:
+ return resident_bytes(module) >> 20
+
+
+def _top_group_mib(module: Any) -> int:
+ from .diffusion_memory import _offload_groups, _storage_nbytes
+ for group in _offload_groups(module) or ():
+ if getattr(group, "offload_leader", None) is module:
+ seen: set = set()
+ total = 0
+ for t in (
+ [p for m in group.modules for p in m.parameters()]
+ + [b for m in group.modules for b in m.buffers()]
+ + list(group.parameters or [])
+ + list(group.buffers or [])
+ ):
+ if id(t) not in seen:
+ seen.add(id(t))
+ total += sum(_storage_nbytes(t))
+ return -(-total // (1 << 20))
+ return 0
+
+
+def room_mib(
+ *,
+ free_mib: int,
+ unused_cache_mib: int,
+ resident_mib_now: int,
+ floor_mib: int,
+ width: int,
+ height: int,
+ frames: int,
+) -> int:
+ from .diffusion_memory import DEFAULT_BASE_OVERHEAD_MIB, estimate_video_runtime_mib
+
+ available = int(free_mib) + max(0, int(unused_cache_mib)) + max(0, int(resident_mib_now))
+ need = (
+ int(floor_mib)
+ + DEFAULT_BASE_OVERHEAD_MIB
+ + estimate_video_runtime_mib(width = width, height = height, num_frames = frames)
+ )
+ return max(0, available - need)
+
+
+def measured_room_mib(
+ *, free_mib: int, unused_cache_mib: int, resident_mib_now: int, peak_extra_mib: int
+) -> int:
+ available = int(free_mib) + max(0, int(unused_cache_mib)) + max(0, int(resident_mib_now))
+ need = int(max(0, int(peak_extra_mib)) * MEASURED_PEAK_MARGIN) + MEASURED_SLACK_MIB
+ return max(0, available - need)
+
+
+def _measured_extra_mib(module: Any, work: int) -> Optional[int]:
+ """The largest recorded peak among completed requests at least as large as ``work``; None when none covers it."""
+ peaks = getattr(module, "_unsloth_video_peaks", None) or {}
+ covering = [extra for w, extra in peaks.items() if w >= work]
+ return max(covering) if covering else None
+
+
+def record_request_peak(pipe: Any, *, logger: Any = None) -> Optional[int]:
+ """Record the request's peak allocation above its start; call only after decode and export."""
+ module = getattr(pipe, "transformer", None)
+ pending = getattr(module, "_unsloth_video_pending", None)
+ if module is None or pending is None:
+ return None
+ try:
+ import torch
+
+ module._unsloth_video_pending = None
+ work, start, dev = pending
+ extra = (int(torch.cuda.max_memory_allocated(dev)) - int(start)) >> 20
+ if extra <= 0:
+ return None
+ peaks = getattr(module, "_unsloth_video_peaks", None)
+ if not isinstance(peaks, dict):
+ peaks = {}
+ module._unsloth_video_peaks = peaks
+ peaks[work] = max(extra, peaks.get(work, 0))
+ _COUNTS["recorded"] += 1
+ if logger is not None:
+ logger.info("video.dit_resident: request peak %d MiB above its start recorded", extra)
+ return extra
+ except Exception: # noqa: BLE001 -- unrecorded keeps the estimate
+ return None
+
+
+def fit_for_request(
+ pipe: Any,
+ *,
+ device: Any,
+ floor_mib: Optional[int],
+ width: int,
+ height: int,
+ frames: int,
+ logger: Any = None,
+) -> Optional[int]:
+ """Re-fit ``pipe.transformer``'s resident set for one request; returns resident MiB or None. Never raises: a
+ failure streams every group again (plain group offload)."""
+ module = getattr(pipe, "transformer", None)
+ if module is None or floor_mib is None:
+ return None
+ try:
+ import torch
+
+ from .diffusion_memory import _keep_groups_resident, release_resident_groups
+
+ dev = torch.device(device)
+ if dev.type != "cuda":
+ return None
+ free, _total = torch.cuda.mem_get_info(dev)
+ unused = int(torch.cuda.memory_reserved(dev)) - int(torch.cuda.memory_allocated(dev))
+ current = resident_mib(module)
+ # an earlier request that never reached record_request_peak (cancelled, failed) leaves nothing measured
+ module._unsloth_video_pending = None
+ work = int(width) * int(height) * int(frames)
+ room = room_mib(
+ free_mib = int(free) >> 20,
+ unused_cache_mib = unused >> 20,
+ resident_mib_now = current,
+ floor_mib = int(floor_mib),
+ width = width,
+ height = height,
+ frames = frames,
+ )
+ measured = _measured_extra_mib(module, work) if measured_enabled() else None
+ if measured is not None:
+ wider = measured_room_mib(
+ free_mib = int(free) >> 20,
+ unused_cache_mib = unused >> 20,
+ resident_mib_now = current,
+ peak_extra_mib = measured,
+ )
+ if wider > room:
+ room = wider
+ _COUNTS["measured_fits"] += 1
+ if not blocks_enabled():
+ top = _top_group_mib(module)
+ room = min(room, top) if top else 0
+ released = promoted = 0
+ over = resident_bytes(module) - (room << 20)
+ if over > 0:
+ restore = release_resident_groups(
+ pipe, -(-over // (1 << 20)), logger, denoisers_only = True, reason = "a larger request"
+ )
+ if restore is not None:
+ released = max(0, current - resident_mib(module))
+ after_release = resident_mib(module)
+ if room > after_release:
+ torch.cuda.empty_cache()
+ _keep_groups_resident(module, room, dev, logger)
+ promoted = max(0, resident_mib(module) - after_release)
+ kept = resident_mib(module)
+ torch.cuda.reset_peak_memory_stats(dev)
+ module._unsloth_video_pending = (work, int(torch.cuda.memory_allocated(dev)), dev)
+ try:
+ # release_resident_groups' restore reads it back; keep it at this request's room
+ module._unsloth_resident_room = room if room > 0 else None
+ except AttributeError:
+ pass
+ _COUNTS["fits"] += 1
+ _COUNTS["promoted_mib"] += promoted
+ _COUNTS["released_mib"] += released
+ _COUNTS["resident_mib"] = kept
+ from .diffusion_memory import _offload_groups
+
+ _COUNTS["groups"] = sum(
+ 1 for g in _offload_groups(module) or () if getattr(g, "_unsloth_resident", False)
+ )
+ if logger is not None and (promoted or released):
+ logger.info(
+ "video.dit_resident: %d MiB of the streamed denoiser resident (%d groups) for %dx%d at %d frames; "
+ "room %d MiB (%s), +%d / -%d MiB; free %d, unused cache %d, floor %d MiB",
+ kept,
+ _COUNTS["groups"],
+ width,
+ height,
+ frames,
+ room,
+ f"measured peak {measured} MiB" if measured is not None else "estimate",
+ promoted,
+ released,
+ int(free) >> 20,
+ unused >> 20,
+ int(floor_mib),
+ )
+ return kept
+ except Exception as exc: # noqa: BLE001 -- residency is a speed-up; stream everything again
+ if logger is not None:
+ logger.warning("video.dit_resident: streaming every group again (%s)", exc)
+ try:
+ from .diffusion_memory import release_resident_groups
+ release_resident_groups(
+ pipe, 1 << 30, logger, denoisers_only = True, reason = "a residency failure"
+ )
+ except Exception: # noqa: BLE001
+ pass
+ return None
diff --git a/studio/backend/tests/test_video_stream_residency.py b/studio/backend/tests/test_video_stream_residency.py
new file mode 100644
index 00000000000..99dc304fd62
--- /dev/null
+++ b/studio/backend/tests/test_video_stream_residency.py
@@ -0,0 +1,297 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Per-request residency of a block-streamed Wan denoiser (video_stream_residency)."""
+
+from __future__ import annotations
+
+import ast
+import types
+from pathlib import Path
+
+import pytest
+
+import core.inference.diffusion_memory as dm
+import core.inference.video_stream_residency as vr
+
+
[email protected](autouse = True)
+def _env(monkeypatch):
+ monkeypatch.delenv(vr.VIDEO_DIT_RESIDENT_ENV, raising = False)
+ monkeypatch.delenv(vr.VIDEO_DIT_RESIDENT_BLOCKS_ENV, raising = False)
+ monkeypatch.delenv(vr.VIDEO_DIT_RESIDENT_MEASURED_ENV, raising = False)
+ monkeypatch.delenv(dm.PARTIAL_RESIDENT_ENV, raising = False)
+
+
+def _applies(**kw):
+ args = dict(is_moe = False, offload_policy = "group", device = "cuda:0")
+ args.update(kw)
+ family = args.pop("family", "wan2.2-ti2v-5b")
+ return vr.applies(family, **args)
+
+
+def test_applies_only_to_group_streamed_single_dit_wan_on_cuda(monkeypatch):
+ assert _applies()
+ assert _applies(device = "cuda")
+ assert not _applies(family = "wan2.2-t2v-a14b", is_moe = True)
+ assert not _applies(family = "ltx-2")
+ assert not _applies(family = "hunyuanvideo-1.5")
+ assert not _applies(offload_policy = "none")
+ assert not _applies(offload_policy = "model")
+ assert not _applies(offload_policy = "streaming")
+ assert not _applies(device = "cpu")
+ assert not _applies(device = "mps")
+ assert not _applies(modular = True)
+ monkeypatch.setenv(vr.VIDEO_DIT_RESIDENT_ENV, "0")
+ assert not _applies()
+
+
+def test_room_shrinks_with_the_clip_and_never_goes_negative():
+ common = dict(free_mib = 9000, unused_cache_mib = 500, resident_mib_now = 0, floor_mib = 2500)
+ short = vr.room_mib(width = 1280, height = 704, frames = 25, **common)
+ long = vr.room_mib(width = 1280, height = 704, frames = 121, **common)
+ assert short > long >= 0
+ # T4 after load: 9000 + 500 - (2500 + 2048 + estimate(25 frames) = 4869) = 83
+ est = dm.estimate_video_runtime_mib(width = 1280, height = 704, num_frames = 25)
+ assert short == 9500 - (2500 + dm.DEFAULT_BASE_OVERHEAD_MIB + est)
+ # already-resident bytes are demotable, so they count as available
+ assert (
+ vr.room_mib(width = 1280, height = 704, frames = 25, **{**common, "resident_mib_now": 4000})
+ == short + 4000
+ )
+ assert vr.room_mib(width = 1280, height = 704, frames = 25, **{**common, "free_mib": 0}) == 0
+
+
+def test_skipped_without_a_floor_or_a_denoiser():
+ assert (
+ vr.fit_for_request(
+ types.SimpleNamespace(), device = "cuda", floor_mib = 1, width = 64, height = 64, frames = 1
+ )
+ is None
+ )
+ net = types.SimpleNamespace()
+ pipe = types.SimpleNamespace(transformer = net, components = {"transformer": net})
+ assert (
+ vr.fit_for_request(pipe, device = "cuda", floor_mib = None, width = 64, height = 64, frames = 1)
+ is None
+ )
+
+
+def test_video_request_path_fits_the_residency_before_the_vram_check():
+ """The generate preflight re-fits the resident set, gated by applies(), before it reads free memory."""
+ src = (Path(__file__).resolve().parents[1] / "core" / "inference" / "video.py").read_text(
+ encoding = "utf-8"
+ )
+ tree = ast.parse(src)
+ calls = [
+ n
+ for n in ast.walk(tree)
+ if isinstance(n, ast.Call)
+ and isinstance(n.func, ast.Attribute)
+ and isinstance(n.func.value, ast.Name)
+ and n.func.value.id == "video_stream_residency"
+ ]
+ names = {c.func.attr for c in calls}
+ assert {"applies", "fit_for_request"} <= names
+ fit = next(c for c in calls if c.func.attr == "fit_for_request")
+ free_reads = [
+ n.lineno
+ for n in ast.walk(tree)
+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "trusted_mem_get_info"
+ ]
+ assert any(line > fit.lineno for line in free_reads)
+
+
+def test_fit_failure_streams_everything_and_never_raises(monkeypatch):
+ released = []
+
+ def boom(*a, **k):
+ raise RuntimeError("no device")
+
+ monkeypatch.setattr(vr, "resident_mib", boom)
+ monkeypatch.setattr(
+ dm, "release_resident_groups", lambda pipe, need, *a, **k: released.append(need)
+ )
+ torch = pytest.importorskip("torch")
+ monkeypatch.setattr(torch.cuda, "mem_get_info", lambda *a: (1 << 30, 1 << 31))
+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda *a: 0)
+ monkeypatch.setattr(torch.cuda, "memory_allocated", lambda *a: 0)
+ net = object()
+ pipe = types.SimpleNamespace(transformer = net, components = {"transformer": net})
+ assert (
+ vr.fit_for_request(pipe, device = "cuda", floor_mib = 1, width = 64, height = 64, frames = 1) is None
+ )
+ assert released
+
+
+# CUDA: real diffusers group offloading with a copy stream.
+
+
+def _streamed_net():
+ torch = pytest.importorskip("torch")
+ if not torch.cuda.is_available():
+ pytest.skip(reason = "CUDA-only: diffusers group offloading with a copy stream")
+ pytest.importorskip("diffusers.hooks")
+ from diffusers.hooks import apply_group_offloading
+
+ class Net(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.proj_in = torch.nn.Linear(64, 1024)
+ self.blocks = torch.nn.ModuleList(
+ torch.nn.Linear(1024, 1024) for _ in range(6)
+ ) # ~4 MiB each
+ self.proj_out = torch.nn.Linear(1024, 64)
+
+ def forward(self, x):
+ x = self.proj_in(x)
+ for block in self.blocks:
+ x = torch.nn.functional.gelu(block(x))
+ return self.proj_out(x)
+
+ torch.manual_seed(0)
+ net = Net()
+ x = torch.randn(3, 64)
+ ref = net.to("cuda")(x.cuda()).cpu()
+ net.to("cpu")
+ apply_group_offloading(
+ net,
+ onload_device = torch.device("cuda"),
+ offload_device = torch.device("cpu"),
+ offload_type = "block_level",
+ num_blocks_per_group = 1,
+ use_stream = True,
+ record_stream = True,
+ non_blocking = True,
+ low_cpu_mem_usage = True,
+ )
+ # the production top-level group: re-pointed to its host copy on offload, as _apply_group_offload leaves it
+ assert dm._skip_top_level_copy_back(net)
+ pipe = types.SimpleNamespace(transformer = net, components = {"transformer": net})
+ return torch, net, pipe, x, ref
+
+
+def _placed(net):
+ return [next(b.parameters()).device.type for b in net.blocks]
+
+
+def _fit(pipe, monkeypatch, room):
+ monkeypatch.setattr(vr, "room_mib", lambda **kw: room)
+ return vr.fit_for_request(pipe, device = "cuda", floor_mib = 1, width = 64, height = 64, frames = 1)
+
+
+def test_base_group_offload_uploads_the_top_level_group_every_call():
+ """What the residency removes: without it the top-level group is back on the host after every call."""
+ torch, net, pipe, x, ref = _streamed_net()
+ for _ in range(2):
+ assert torch.equal(net(x.cuda()).cpu(), ref)
+ assert next(net.proj_in.parameters()).device.type == "cpu"
+ assert _placed(net) == ["cpu"] * 6
+
+
+def test_top_level_and_block_prefix_stay_resident_bit_identical(monkeypatch):
+ torch, net, pipe, x, ref = _streamed_net()
+ kept = _fit(pipe, monkeypatch, 1024)
+ assert kept and kept >= 24
+ for _ in range(3): # first call traces the prefetch order, later ones prefetch
+ assert torch.equal(net(x.cuda()).cpu(), ref)
+ assert next(net.proj_in.parameters()).device.type == "cuda"
+ assert next(net.proj_out.parameters()).device.type == "cuda"
+ assert _placed(net) == ["cuda"] * 6
+ assert vr.counts()["resident_mib"] == kept
+
+
+def test_prefix_is_sized_to_the_room_and_refit_per_request(monkeypatch):
+ torch, net, pipe, x, ref = _streamed_net()
+ block_mib = 4
+ _fit(pipe, monkeypatch, 1 + 3 * block_mib + 1) # top-level (~0.5 MiB) + 3 blocks
+ assert _placed(net) == ["cuda"] * 3 + ["cpu"] * 3
+ assert next(net.proj_in.parameters()).device.type == "cuda"
+ for _ in range(2):
+ assert torch.equal(net(x.cuda()).cpu(), ref)
+ # a longer clip: less room, the prefix shrinks from its end
+ _fit(pipe, monkeypatch, 1 + block_mib + 1)
+ assert _placed(net) == ["cuda"] + ["cpu"] * 5
+ for _ in range(2):
+ assert torch.equal(net(x.cuda()).cpu(), ref)
+ # a short clip again: grows back
+ _fit(pipe, monkeypatch, 1024)
+ assert _placed(net) == ["cuda"] * 6
+ assert torch.equal(net(x.cuda()).cpu(), ref)
+ # no room: everything streams, as plain group offloading
+ _fit(pipe, monkeypatch, 0)
+ assert _placed(net) == ["cpu"] * 6
+ assert next(net.proj_in.parameters()).device.type == "cpu"
+ for _ in range(2):
+ assert torch.equal(net(x.cuda()).cpu(), ref)
+ assert next(net.proj_in.parameters()).device.type == "cpu"
+
+
+def test_blocks_kill_switch_keeps_only_the_top_level_group(monkeypatch):
+ torch, net, pipe, x, ref = _streamed_net()
+ monkeypatch.setenv(vr.VIDEO_DIT_RESIDENT_BLOCKS_ENV, "0")
+ _fit(pipe, monkeypatch, 1024)
+ assert _placed(net) == ["cpu"] * 6
+ for _ in range(2):
+ assert torch.equal(net(x.cuda()).cpu(), ref)
+ assert next(net.proj_in.parameters()).device.type == "cuda"
+
+
+def test_measured_room_and_cover_lookup():
+ common = dict(free_mib = 9000, unused_cache_mib = 500, resident_mib_now = 2000)
+ assert (
+ vr.measured_room_mib(peak_extra_mib = 2000, **common)
+ == 11500 - int(2000 * vr.MEASURED_PEAK_MARGIN) - vr.MEASURED_SLACK_MIB
+ )
+ assert vr.measured_room_mib(peak_extra_mib = 20000, **common) == 0
+ net = types.SimpleNamespace(_unsloth_video_peaks = {100: 1500, 300: 2500})
+ assert vr._measured_extra_mib(net, 100) == 2500 # every recorded request at least this large
+ assert vr._measured_extra_mib(net, 200) == 2500
+ assert (
+ vr._measured_extra_mib(net, 301) is None
+ ) # larger than anything measured: the estimate sizes it
+ assert vr._measured_extra_mib(types.SimpleNamespace(), 1) is None
+
+
+def test_only_a_completed_request_is_recorded(monkeypatch):
+ torch = pytest.importorskip("torch")
+ monkeypatch.setattr(torch.cuda, "max_memory_allocated", lambda *a: 5000 << 20)
+ net = types.SimpleNamespace(_unsloth_video_pending = (100, 1000 << 20, "cuda"))
+ pipe = types.SimpleNamespace(transformer = net)
+ assert vr.record_request_peak(pipe) == 4000
+ assert net._unsloth_video_peaks == {100: 4000}
+ assert net._unsloth_video_pending is None
+ assert vr.record_request_peak(pipe) is None # recorded once per fit
+
+
+def test_video_request_path_records_the_peak_after_export():
+ src = (Path(__file__).resolve().parents[1] / "core" / "inference" / "video.py").read_text(
+ encoding = "utf-8"
+ )
+ record = src.index("video_stream_residency.record_request_peak(")
+ assert src.index("mp4_bytes = self._encode_mp4(") < record < src.index('"mp4_bytes": mp4_bytes')
+
+
+def test_measured_peak_widens_the_next_request(monkeypatch):
+ torch, net, pipe, x, ref = _streamed_net()
+ est = {"room": 1 + 4 + 1} # top-level + 1 block from the estimate
+ monkeypatch.setattr(vr, "room_mib", lambda **kw: est["room"])
+ fit = lambda: vr.fit_for_request(
+ pipe, device = "cuda", floor_mib = 1, width = 64, height = 64, frames = 1
+ ) # noqa: E731
+ fit()
+ assert _placed(net) == ["cuda"] + ["cpu"] * 5
+ assert torch.equal(net(x.cuda()).cpu(), ref)
+ vr.record_request_peak(pipe)
+ net._unsloth_video_peaks = {64 * 64 * 1: 1}
+ fit() # the measured room on a large card holds every block
+ assert _placed(net) == ["cuda"] * 6
+ assert torch.equal(net(x.cuda()).cpu(), ref)
+ monkeypatch.setenv(vr.VIDEO_DIT_RESIDENT_MEASURED_ENV, "0")
+ fit()
+ assert _placed(net) == ["cuda"] + ["cpu"] * 5
+ assert torch.equal(net(x.cuda()).cpu(), ref)
+ # a larger request than anything measured is sized by the estimate
+ monkeypatch.delenv(vr.VIDEO_DIT_RESIDENT_MEASURED_ENV)
+ vr.fit_for_request(pipe, device = "cuda", floor_mib = 1, width = 64, height = 64, frames = 2)
+ assert _placed(net) == ["cuda"] + ["cpu"] * 5