Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio video: price deferred pins at their bytes so a returning Wan load pins its text encoder (#15612)

Daniel Han702378d916e2
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index 78a2629e41d..4766e85686a 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -6089,6 +6089,24 @@ def _module_registered_mib(module: Any) -> int:
return _module_host_mib(module)
+def _module_offload_registered_mib(module: Any) -> int:
+ """Registered MiB a group offload pins: page-rounded, counted once per owning submodule (UMT5's tied embedding pins twice)."""
+ try:
+ total = 0
+ for sub in module.modules():
+ seen: set[int] = set()
+ for tensor in list(sub.parameters(recurse = False)) + list(sub.buffers(recurse = False)):
+ if id(tensor) in seen:
+ continue
+ seen.add(id(tensor))
+ total += sum(
+ _page_bytes(nbytes) for nbytes in _storage_nbytes(tensor) if nbytes > 0
+ )
+ return -(-total // (1024 * 1024))
+ except Exception: # noqa: BLE001 - priced as the per-tensor pin
+ return _module_host_mib(module)
+
+
def _unique_tensors(module: Any) -> list:
seen: set[int] = set()
out = []
@@ -6299,6 +6317,9 @@ ENCODER_BACKGROUND_PIN_ENV = "UNSLOTH_DIFFUSION_ENCODER_BACKGROUND_PIN"
PARTIAL_ENCODER_PIN_ENV = "UNSLOTH_DIFFUSION_PARTIAL_ENCODER_PIN"
_ENCODER_PIN_REQUEST_ATTR = "_unsloth_background_encoder_pin_requested"
_PARTIAL_PIN_MIN_MIB = 256
+# "0": old behaviour (power-of-two pin price, no partial encoder prefix).
+VIDEO_EXACT_PIN_PLAN_ENV = "UNSLOTH_VIDEO_EXACT_PIN_PLAN"
+_CAST_WRITES_PENDING_ATTR = "_unsloth_cast_writes_pending"
def request_background_encoder_pins(pipe: Any) -> None:
@@ -6330,6 +6351,65 @@ def _encoder_background_pin_wanted(pipe: Any, device: Any = None) -> bool:
)
+def note_cast_cache_writes(pipe: Any, writes: Any) -> None:
+ """Record the cast-cache writes a first load in a Studio home deferred; the pin plan reads them (``done()``)."""
+ try:
+ setattr(pipe, _CAST_WRITES_PENDING_ATTR, writes)
+ except Exception: # noqa: BLE001 - a pipe refusing attributes keeps the exact plan
+ pass
+
+
+def _cast_cache_write_pending(pipe: Any) -> bool:
+ writes = getattr(pipe, _CAST_WRITES_PENDING_ATTR, None)
+ if writes is None:
+ return False
+ try:
+ return not writes.done()
+ except Exception: # noqa: BLE001 - unreadable: treat as still writing
+ return True
+
+
+def _video_exact_pins(
+ pipe: Any,
+ device: Any = None,
+ logger: Any = None,
+) -> bool:
+ """Video load priced at page-rounded bytes; a first load still writing its cast cache keeps the default plan (extra pins cost it 3-11 s)."""
+ if not (
+ bool(getattr(pipe, _FAST_PIN_REQUEST_ATTR, False))
+ and not _env_off(VIDEO_EXACT_PIN_PLAN_ENV)
+ and _fast_pin_supported(device)
+ ):
+ return False
+ if _cast_cache_write_pending(pipe):
+ if logger is not None:
+ logger.info(
+ "diffusion.memory: this load writes its cast cache (first load in this home): its pins keep the "
+ "default plan, no encoder prefix"
+ )
+ return False
+ return True
+
+
+def _late_encoder_pin_cap(
+ pipe: Any,
+ module: Any,
+ device: Any = None,
+) -> Callable[[], int]:
+ """Encoder pin cap read when the pinner starts, not at plan time (the load still holds source shards then), minus earlier pinners' unplaced bytes."""
+
+ def cap() -> int:
+ budget = _streamed_pin_leftover_mib(0, False, device)
+ owed = 0
+ for pinner in list(getattr(pipe, _PENDING_PINS_ATTR, None) or ()):
+ if pinner.module is module:
+ break
+ owed += pinner.outstanding_bytes()
+ return max(0, (int(budget) << 20) - owed)
+
+ return cap
+
+
def _streamed_pin_leftover_mib(
transformer_mib: int,
pin_transformer: bool,
@@ -6561,8 +6641,11 @@ class _GroupPinner:
import threading
self.fast = bool(fast)
- # Pin whole groups in order while they fit; the rest stay pageable (diffusers pins them per onload).
- self.byte_cap = None if byte_cap is None else max(0, int(byte_cap))
+ # Pin whole groups in order while they fit; the rest stay pageable.
+ if byte_cap is None or callable(byte_cap):
+ self.byte_cap = byte_cap
+ else:
+ self.byte_cap = max(0, int(byte_cap))
self.module = module
self.label = type(module).__name__
self.groups = groups
@@ -6634,6 +6717,22 @@ class _GroupPinner:
except Exception: # noqa: BLE001 - registration refused: the allocator's pin for this tensor
return src.pin_memory()
+ def outstanding_bytes(self) -> int:
+ """Bytes this pinner may still place: its unpinned groups (placed ones already left that sum), within what
+ its byte cap has left."""
+ try:
+ need = sum(
+ _page_bytes(src.nbytes)
+ for group in self.groups
+ for _t, src in self._unpinned(group)
+ )
+ except Exception: # noqa: BLE001 - unreadable: owes nothing
+ return 0
+ cap = self.byte_cap
+ if callable(cap) or cap is None:
+ return need
+ return min(need, max(0, int(cap) - int(self.pinned)))
+
def _capped_groups(self) -> list:
if self.byte_cap is None:
return list(self.groups)
@@ -6656,6 +6755,14 @@ class _GroupPinner:
failed = None
pool = None
try:
+ if callable(self.byte_cap):
+ self.byte_cap = max(0, int(self.byte_cap()))
+ if self.logger is not None:
+ self.logger.info(
+ "diffusion.memory: %s pins up to %.1f GiB of its offload groups (cap read at pin start)",
+ self.label,
+ self.byte_cap / 2**30,
+ )
if self.byte_cap is not None:
kept = self._capped_groups()
kept_ids = {id(g) for g in kept}
@@ -7207,6 +7314,16 @@ def _apply_group_offload(
)
# an up-front pin packs into slabs (_apply_group_offloading_packed); a deferred one pins per tensor
pin_mib = _module_host_mib if defer else _module_pin_mib
+ # Power-of-two pricing read Wan's 12.5 GiB UMT5 as 17 GiB, so it never pinned on 31-51 GiB hosts.
+ exact_defer = defer and _video_exact_pins(pipe, onload, logger)
+ if exact_defer:
+ _host_price = pin_mib
+
+ def pin_mib(module: Any) -> int:
+ if _torchao_weight_classes(module):
+ return _host_price(module)
+ return _module_offload_registered_mib(module)
+
# Image loads: only the streamed encoders defer (registered pins, priced at weight bytes); torchao ones cannot.
encoders_defer = (
not defer
@@ -7242,6 +7359,18 @@ def _apply_group_offload(
pin_streamed = (False, False)
gkwargs["low_cpu_mem_usage"] = True
encoder_cap_mib = 0
+ late_prefix = (
+ exact_defer
+ and stream_text_encoders
+ and not pin_streamed[1]
+ and not small_host
+ and not _env_off(PARTIAL_ENCODER_PIN_ENV)
+ )
+ if late_prefix and logger is not None:
+ logger.info(
+ "diffusion.memory: streamed encoders pin the prefix of their offload groups that fits the host RAM "
+ "free when the load returns"
+ )
if encoders_defer and stream_text_encoders and not pin_streamed[1] and not small_host:
encoder_cap_mib = _streamed_pin_leftover_mib(dit_pin_mib, pin_streamed[0], onload)
if encoder_cap_mib and logger is not None:
@@ -7304,6 +7433,16 @@ def _apply_group_offload(
if defer and whole:
apply_group_offloading(module, **{**ekwargs, "low_cpu_mem_usage": True})
_defer_pinning(pipe, module, onload, logger)
+ elif late_prefix and use_stream and not _torchao_weight_classes(module):
+ apply_group_offloading(module, **{**ekwargs, "low_cpu_mem_usage": True})
+ _defer_pinning(
+ pipe,
+ module,
+ onload,
+ logger,
+ fast = True,
+ byte_cap = _late_encoder_pin_cap(pipe, module, onload),
+ )
elif _encoder_defers(module) and (whole or encoder_left_mib > 0):
apply_group_offloading(module, **{**ekwargs, "low_cpu_mem_usage": True})
cap = None if whole else int(encoder_left_mib) << 20
@@ -7957,6 +8096,15 @@ def _apply_streaming_offload(
)
# an up-front pin packs into slabs (_apply_group_offloading_packed); a deferred one pins per tensor
pin_mib = _module_host_mib if defer else _module_pin_mib
+ exact_defer = defer and _video_exact_pins(pipe, onload, logger)
+ if exact_defer:
+ _host_price = pin_mib
+
+ def pin_mib(module: Any) -> int:
+ if _torchao_weight_classes(module):
+ return _host_price(module)
+ return _module_offload_registered_mib(module)
+
encoders_defer = not defer and prefetch and _encoder_background_pin_wanted(pipe, onload)
def _encoder_defers(name: Any, module: Any, offload_type: str) -> bool:
@@ -7981,6 +8129,19 @@ def _apply_streaming_offload(
pin_dits, pin_encoders = _streamed_pin_plan(
dit_pin_mib, leaf_mib, logger, device = onload
)
+ late_prefix = (
+ exact_defer
+ and prefetch
+ and not pin_encoders
+ and not getattr(pipe, "_unsloth_small_host", None)
+ and not _env_off(PARTIAL_ENCODER_PIN_ENV)
+ and host_pinning_allowed(onload)
+ )
+ if late_prefix and logger is not None:
+ logger.info(
+ "diffusion.memory: streamed encoders pin the prefix of their offload groups that fits the host RAM "
+ "free when the load returns"
+ )
encoder_left_mib = 0
if encoders_defer and not pin_encoders:
encoder_left_mib = _streamed_pin_leftover_mib(dit_pin_mib, pin_dits, onload)
@@ -8021,6 +8182,23 @@ def _apply_streaming_offload(
_apply_group_offloading_packed(apply_group_offloading, module, kwargs, logger)
if pin and defer and kwargs.get("use_stream") and kwargs.get("low_cpu_mem_usage"):
_defer_pinning(pipe, module, onload, logger)
+ elif (
+ late_prefix
+ and not pin
+ and offload_type != "block_level"
+ and str(comp_name).startswith("text_encoder")
+ and not _torchao_weight_classes(module)
+ and kwargs.get("use_stream")
+ and kwargs.get("low_cpu_mem_usage")
+ ):
+ _defer_pinning(
+ pipe,
+ module,
+ onload,
+ logger,
+ fast = True,
+ byte_cap = _late_encoder_pin_cap(pipe, module, onload),
+ )
elif enc_defer and kwargs.get("use_stream") and kwargs.get("low_cpu_mem_usage"):
cap = None if pin else int(encoder_left_mib) << 20
_defer_pinning(pipe, module, onload, logger, fast = True, byte_cap = cap)
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 7c37c327455..df387ec92c4 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -8941,6 +8941,11 @@ class VideoBackend:
# Names only: the write reads the live tensors once the load is published.
cast_writes = defer_writes(pipe, cast_missed, logger = logger)
+ if cast_writes is not None:
+ from .diffusion_memory import note_cast_cache_writes
+
+ # the pin plan below keeps its default price while this first load writes its cast cache
+ note_cast_cache_writes(pipe, cast_writes)
else:
transformer_cls = getattr(diffusers, fam.transformer_class)
# checkpoint_path was resolved (and downloaded) by the memory-planning branch above.
diff --git a/studio/backend/tests/test_video_encoder_exact_pin.py b/studio/backend/tests/test_video_encoder_exact_pin.py
new file mode 100644
index 00000000000..0a1ea924d66
--- /dev/null
+++ b/studio/backend/tests/test_video_encoder_exact_pin.py
@@ -0,0 +1,425 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Page-rounded video pin pricing and the late encoder prefix pin."""
+
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+import core.inference.diffusion_memory as dm
+
+_CUDA = torch.cuda.is_available()
+_EXACT_ENV = getattr(dm, "VIDEO_EXACT_PIN_PLAN_ENV", "UNSLOTH_VIDEO_EXACT_PIN_PLAN")
+
+
+def host(monkeypatch):
+ for name in (
+ dm.GROUP_OFFLOAD_PIN_ENV,
+ dm.PARTIAL_ENCODER_PIN_ENV,
+ dm.ENCODER_BACKGROUND_PIN_ENV,
+ dm.BACKGROUND_PIN_ENV,
+ _EXACT_ENV,
+ ):
+ monkeypatch.delenv(name, raising = False)
+ monkeypatch.setattr(dm, "host_pinning_allowed", lambda *a, **k: True)
+ monkeypatch.setattr(dm, "_pinned_memory_capped", lambda: False)
+ monkeypatch.setattr(dm, "_PARTIAL_PIN_MIN_MIB", 1)
+ state = {"mib": 0}
+ monkeypatch.setattr(dm, "_pin_budget_mib", lambda: state["mib"])
+
+ def set_mib(mib):
+ state["mib"] = mib
+
+ return set_mib
+
+
+class _Pipe:
+ pass
+
+
+def _encoder():
+ torch.manual_seed(0)
+ # 4,500,000 bytes of bf16 weight per Linear: 8 MiB as a power of two, 1099 pages registered
+ return torch.nn.Sequential(*[torch.nn.Linear(1500, 1500) for _ in range(4)]).to(torch.bfloat16)
+
+
+def _video_pipe():
+ pipe = _Pipe()
+ pipe.transformer = torch.nn.Linear(8, 8).to(torch.bfloat16)
+ pipe.text_encoder = _encoder()
+ pipe.components = {"transformer": pipe.transformer, "text_encoder": pipe.text_encoder}
+ dm.request_background_pins(pipe)
+ dm.request_fast_pins(pipe)
+ return pipe
+
+
+def _host_copies(module):
+ return [t for g in dm._offload_groups(module) for t in g.cpu_param_dict.values()]
+
+
+def test_the_two_prices_differ_for_this_encoder():
+ enc = _encoder()
+ assert dm._module_host_mib(enc) == 32
+ assert dm._module_registered_mib(enc) == 18
+
+
+def test_a_tied_embedding_is_priced_once_per_offload_group_that_copies_it():
+ class Tied(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.shared = torch.nn.Embedding(1024, 1024).to(torch.bfloat16)
+ self.embed_tokens = self.shared
+ self.encoder = torch.nn.Module()
+ self.encoder.embed_tokens = self.shared # UMT5: one tensor, two owning submodules
+ self.proj = torch.nn.Linear(1024, 1024, bias = False).to(torch.bfloat16)
+
+ m = Tied()
+ assert dm._module_registered_mib(m) == 4
+ assert dm._module_offload_registered_mib(m) == 4 # modules() visits the shared Embedding once
+ m.encoder.embed_tokens = torch.nn.Embedding(1024, 1024).to(torch.bfloat16)
+ m.encoder.embed_tokens.weight = (
+ m.shared.weight
+ ) # tied weight, separate module: two leaf groups copy it
+ assert dm._module_registered_mib(m) == 4
+ assert dm._module_offload_registered_mib(m) == 6
+
+
+def test_the_kill_switch_and_an_unsupported_registered_pin_turn_the_exact_plan_off(monkeypatch):
+ monkeypatch.delenv(_EXACT_ENV, raising = False)
+ monkeypatch.setattr(dm, "_fast_pin_supported", lambda *a, **k: True)
+ pipe = types.SimpleNamespace()
+ assert not dm._video_exact_pins(pipe)
+ dm.request_fast_pins(pipe)
+ assert dm._video_exact_pins(pipe)
+ monkeypatch.setenv(_EXACT_ENV, "0")
+ assert not dm._video_exact_pins(pipe)
+ monkeypatch.delenv(_EXACT_ENV)
+ monkeypatch.setattr(dm, "_fast_pin_supported", lambda *a, **k: False)
+ assert not dm._video_exact_pins(pipe)
+
+
+def test_the_late_cap_reads_the_budget_at_pin_start_less_what_earlier_pinners_owe(host):
+ mine = torch.nn.Linear(4, 4)
+ pipe = types.SimpleNamespace()
+ earlier = types.SimpleNamespace(module = torch.nn.Linear(4, 4), outstanding_bytes = lambda: 2 << 20)
+ me = types.SimpleNamespace(module = mine, outstanding_bytes = lambda: 99 << 20)
+ setattr(pipe, dm._PENDING_PINS_ATTR, [earlier, me])
+ cap = dm._late_encoder_pin_cap(pipe, mine)
+ host(0)
+ assert cap() == 0
+ host(10)
+ assert cap() == (10 << 20) - (2 << 20)
+
+
+def test_an_earlier_pinner_owes_its_unplaced_groups_once(monkeypatch):
+ """Placed groups already leave the unpinned sum: the cap must not subtract them a second time."""
+ monkeypatch.setattr(torch.Tensor, "is_pinned", lambda self: getattr(self, "_fake_pin", False))
+ groups, srcs = [], []
+ for _ in range(2):
+ w = torch.nn.Parameter(torch.zeros(4 << 20, dtype = torch.uint8), requires_grad = False)
+ src = w.data
+ groups.append(types.SimpleNamespace(cpu_param_dict = {w: src}))
+ srcs.append(src)
+ pinner = dm._GroupPinner(torch.nn.Module(), groups, torch.device("cpu"))
+ assert pinner.outstanding_bytes() == 8 << 20
+ srcs[0]._fake_pin = True # the first group is placed
+ pinner.pinned = 4 << 20
+ assert pinner.outstanding_bytes() == 4 << 20
+ capped = dm._GroupPinner(torch.nn.Module(), groups, torch.device("cpu"), byte_cap = 6 << 20)
+ capped.pinned = 4 << 20
+ assert (
+ capped.outstanding_bytes() == 2 << 20
+ ) # what the cap has left, not the whole remaining group
+
+
+def test_a_callable_cap_is_read_when_the_pinner_runs(monkeypatch):
+ monkeypatch.setattr(torch.Tensor, "is_pinned", lambda self: getattr(self, "_fake_pin", False))
+
+ def pin_memory(self):
+ out = self.clone()
+ out._fake_pin = True
+ return out
+
+ monkeypatch.setattr(torch.Tensor, "pin_memory", pin_memory)
+ groups = []
+ for n in (100, 5000, 300):
+ w = torch.nn.Parameter(torch.zeros(n, dtype = torch.uint8), requires_grad = False)
+ groups.append(types.SimpleNamespace(cpu_param_dict = {w: w.data}))
+ budget = {"bytes": 0}
+ pinner = dm._GroupPinner(
+ torch.nn.Module(), groups, torch.device("cpu"), byte_cap = lambda: budget["bytes"]
+ )
+ budget["bytes"] = 3 * 4096 + 10 # 1 + 2 pages fit, the third group does not
+ pinner.start()
+ pinner.join(30)
+ assert pinner.byte_cap == 3 * 4096 + 10
+ assert [all(t.is_pinned() for t in g.cpu_param_dict.values()) for g in groups] == [
+ True,
+ True,
+ False,
+ ]
+
+
[email protected](not _CUDA, reason = "needs a CUDA device")
+def test_a_video_encoder_that_fits_its_registered_bytes_pins_whole(host, monkeypatch):
+ pytest.importorskip("diffusers")
+ x = torch.randn(4, 1500, dtype = torch.bfloat16, device = "cuda")
+ with torch.no_grad():
+ ref = _encoder().to("cuda")(x)
+ host(20) # 18 MiB registered fits; the 32 MiB power-of-two price did not
+ pipe = _video_pipe()
+ assert dm._apply_group_offload(
+ pipe, "cuda", None, stream_text_encoders = True, stream_transformer = False
+ )
+ assert dm.start_background_pins(pipe) == 1
+ dm.finish_background_pins(pipe)
+ assert all(t.is_pinned() for t in _host_copies(pipe.text_encoder))
+ with torch.no_grad():
+ assert torch.equal(pipe.text_encoder(x), ref)
+ dm.stop_background_pins(pipe)
+
+
[email protected](not _CUDA, reason = "needs a CUDA device")
+def test_a_video_encoder_too_big_at_the_plan_pins_the_prefix_free_at_pin_start(host, monkeypatch):
+ pytest.importorskip("diffusers")
+ x = torch.randn(4, 1500, dtype = torch.bfloat16, device = "cuda")
+ with torch.no_grad():
+ ref = _encoder().to("cuda")(x)
+ host(2) # the plan: nothing fits (the load still holds its source shards)
+ pipe = _video_pipe()
+ assert dm._apply_group_offload(
+ pipe, "cuda", None, stream_text_encoders = True, stream_transformer = False
+ )
+ assert not any(t.is_pinned() for t in _host_copies(pipe.text_encoder))
+ host(10) # the load returned: 10 MiB free, two 4.3 MiB weight groups (and their bias pages) fit
+ assert dm.start_background_pins(pipe) == 1
+ dm.finish_background_pins(pipe)
+ pinned = [t.is_pinned() for t in _host_copies(pipe.text_encoder)]
+ assert pinned == [True] * 4 + [False] * 4
+ with torch.no_grad():
+ assert torch.equal(pipe.text_encoder(x), ref)
+ dm.stop_background_pins(pipe)
+
+
[email protected](not _CUDA, reason = "needs a CUDA device")
+def test_the_kill_switch_keeps_the_power_of_two_price_and_no_prefix(host, monkeypatch):
+ pytest.importorskip("diffusers")
+ monkeypatch.setenv(_EXACT_ENV, "0")
+ host(20)
+ pipe = _video_pipe()
+ assert dm._apply_group_offload(
+ pipe, "cuda", None, stream_text_encoders = True, stream_transformer = False
+ )
+ assert dm.start_background_pins(pipe) == 0
+ assert not any(t.is_pinned() for t in _host_copies(pipe.text_encoder))
+
+
+class _Blocks(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.proj_in = torch.nn.Linear(8, 8)
+ self.blocks = torch.nn.ModuleList(torch.nn.Linear(8, 8) for _ in range(2))
+
+ def forward(self, x):
+ x = self.proj_in(x)
+ for block in self.blocks:
+ x = x + block(x)
+ return x
+
+
+def _streaming_video_pipe():
+ pipe = _Pipe()
+ pipe.transformer = _Blocks().to(torch.bfloat16)
+ pipe.text_encoder = _encoder()
+ pipe.components = {"transformer": pipe.transformer, "text_encoder": pipe.text_encoder}
+ dm.request_background_pins(pipe)
+ dm.request_fast_pins(pipe)
+ return pipe
+
+
[email protected](not _CUDA, reason = "needs a CUDA device")
+def test_the_streaming_tier_pins_a_video_encoder_that_fits_its_registered_bytes(host, monkeypatch):
+ pytest.importorskip("diffusers")
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_STREAMING_PREFETCH", raising = False)
+ x = torch.randn(4, 1500, dtype = torch.bfloat16, device = "cuda")
+ with torch.no_grad():
+ ref = _encoder().to("cuda")(x)
+ host(20) # 18 MiB registered + a few KiB of blocks fit; the 32 MiB power-of-two price did not
+ pipe = _streaming_video_pipe()
+ dm._apply_streaming_offload(pipe, "cuda", None)
+ assert dm.start_background_pins(pipe) == 2
+ dm.finish_background_pins(pipe)
+ assert all(t.is_pinned() for t in _host_copies(pipe.text_encoder))
+ with torch.no_grad():
+ assert torch.equal(pipe.text_encoder(x), ref)
+ dm.stop_background_pins(pipe)
+
+
[email protected](not _CUDA, reason = "needs a CUDA device")
+def test_the_streaming_tier_pins_the_prefix_free_at_pin_start(host, monkeypatch):
+ pytest.importorskip("diffusers")
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_STREAMING_PREFETCH", raising = False)
+ x = torch.randn(4, 1500, dtype = torch.bfloat16, device = "cuda")
+ with torch.no_grad():
+ ref = _encoder().to("cuda")(x)
+ host(2) # the plan: the blocks fit, the encoder does not
+ pipe = _streaming_video_pipe()
+ dm._apply_streaming_offload(pipe, "cuda", None)
+ assert not any(t.is_pinned() for t in _host_copies(pipe.text_encoder))
+ host(10) # once the load returned: two 4.3 MiB weight groups (and their bias pages) fit
+ assert dm.start_background_pins(pipe) == 2
+ dm.finish_background_pins(pipe)
+ pinned = [t.is_pinned() for t in _host_copies(pipe.text_encoder)]
+ assert pinned == [True] * 4 + [False] * 4
+ with torch.no_grad():
+ assert torch.equal(pipe.text_encoder(x), ref)
+ dm.stop_background_pins(pipe)
+
+
[email protected](not _CUDA, reason = "needs a CUDA device")
+def test_the_streaming_tier_kill_switch_pins_no_encoder(host, monkeypatch):
+ pytest.importorskip("diffusers")
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_STREAMING_PREFETCH", raising = False)
+ monkeypatch.setenv(_EXACT_ENV, "0")
+ host(20)
+ pipe = _streaming_video_pipe()
+ dm._apply_streaming_offload(pipe, "cuda", None)
+ dm.start_background_pins(pipe)
+ dm.finish_background_pins(pipe)
+ assert not any(t.is_pinned() for t in _host_copies(pipe.text_encoder))
+ dm.stop_background_pins(pipe)
+
+
+class _Writes:
+ def __init__(self, finished = False):
+ self.finished = finished
+
+ def done(self):
+ return self.finished
+
+
+def _plan_only(monkeypatch, mib):
+ """CPU: run the pin plan of a video load with every device-side step stubbed; returns the deferred pins."""
+ pytest.importorskip("diffusers.hooks")
+ import sys
+
+ deferred = []
+
+ def apply_group_offloading(
+ module,
+ onload_device = None,
+ offload_device = None,
+ offload_type = None,
+ num_blocks_per_group = None,
+ use_stream = False,
+ non_blocking = False,
+ record_stream = False,
+ low_cpu_mem_usage = False,
+ ):
+ return None
+
+ monkeypatch.setattr(
+ sys.modules["diffusers.hooks"], "apply_group_offloading", apply_group_offloading
+ )
+ for name in (
+ _EXACT_ENV,
+ dm.PARTIAL_ENCODER_PIN_ENV,
+ dm.GROUP_OFFLOAD_PIN_ENV,
+ dm.BACKGROUND_PIN_ENV,
+ ):
+ monkeypatch.delenv(name, raising = False)
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_STREAMING_PREFETCH", raising = False)
+ for name, value in (
+ ("host_pinning_allowed", True),
+ ("_fast_pin_supported", True),
+ ("_background_pin_enabled", True),
+ ("_streaming_prefetch_enabled", True),
+ ("_pinned_memory_capped", False),
+ ("_pin_top_level_group", True),
+ ):
+ monkeypatch.setattr(dm, name, lambda *a, _v = value, **k: _v)
+ for name in (
+ "install_group_offload_buffer_restore",
+ "install_group_offload_hooks_eager",
+ "install_group_pin_wait",
+ "install_group_prefetch",
+ "_skip_top_level_copy_back",
+ "_pin_vision_embedding_device",
+ ):
+ monkeypatch.setattr(dm, name, lambda *a, **k: None)
+ monkeypatch.setattr(
+ dm,
+ "_apply_group_offloading_packed",
+ lambda apply, module, kwargs, logger: apply(module, **kwargs),
+ )
+ monkeypatch.setattr(dm, "_pin_budget_mib", lambda: mib)
+ monkeypatch.setattr(
+ dm,
+ "_defer_pinning",
+ lambda pipe, module, device, logger, *, fast = None, byte_cap = None: deferred.append(
+ (module, byte_cap)
+ ),
+ )
+ return deferred
+
+
+def _te_deferrals(pipe, deferred):
+ return [cap for module, cap in deferred if module is pipe.text_encoder]
+
+
[email protected]("tier", ["group", "streaming"])
+def test_a_first_load_writing_its_cast_cache_pins_no_encoder_prefix(monkeypatch, tier):
+ deferred = _plan_only(
+ monkeypatch, 10
+ ) # 18 MiB registered, 32 MiB power-of-two: neither fits whole
+ pipe = _streaming_video_pipe()
+ dm.note_cast_cache_writes(pipe, _Writes(finished = False))
+ if tier == "group":
+ dm._apply_group_offload(
+ pipe, "cuda", None, stream_text_encoders = True, stream_transformer = True
+ )
+ else:
+ dm._apply_streaming_offload(pipe, "cuda", None)
+ assert _te_deferrals(pipe, deferred) == [] # what main does: the encoder streams unpinned
+
+
[email protected]("tier", ["group", "streaming"])
[email protected]("writes", [None, "finished"])
+def test_a_returning_load_pins_the_encoder_prefix(monkeypatch, tier, writes):
+ deferred = _plan_only(monkeypatch, 10)
+ pipe = _streaming_video_pipe()
+ if writes:
+ dm.note_cast_cache_writes(pipe, _Writes(finished = True))
+ if tier == "group":
+ dm._apply_group_offload(
+ pipe, "cuda", None, stream_text_encoders = True, stream_transformer = True
+ )
+ else:
+ dm._apply_streaming_offload(pipe, "cuda", None)
+ caps = _te_deferrals(pipe, deferred)
+ assert len(caps) == 1 and callable(caps[0]) # the late prefix cap
+
+
+def test_the_pending_write_gate_reads_done_and_survives_a_broken_record():
+ pipe = types.SimpleNamespace()
+ assert not dm._cast_cache_write_pending(pipe)
+ dm.note_cast_cache_writes(pipe, _Writes(finished = False))
+ assert dm._cast_cache_write_pending(pipe)
+ dm.note_cast_cache_writes(pipe, _Writes(finished = True))
+ assert not dm._cast_cache_write_pending(pipe)
+ dm.note_cast_cache_writes(pipe, object()) # no done(): treated as still writing
+ assert dm._cast_cache_write_pending(pipe)
+
+
+def test_the_video_load_records_its_cast_writes_before_the_pin_plan():
+ from pathlib import Path
+
+ src = (Path(dm.__file__).parent / "video.py").read_text()
+ deferred = src.index("cast_writes = defer_writes(pipe, cast_missed")
+ noted = src.index("note_cast_cache_writes(pipe, cast_writes)")
+ planned = src.index("offload_policy, vae_tiling = apply_memory_plan(")
+ assert deferred < noted < planned