Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: stop the first Wan2.2-A14B render from waiting for the second expert's pin (#15906)

Daniel Han9258bfc73208
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index 5ecf1b9a38d..50d813a8a75 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -6747,11 +6747,14 @@ class _GroupPinner:
self.device = device
self.logger = logger
self.pinned = 0
+ self.copied = 0
self.waited_s = 0.0
self._done = {id(g): threading.Event() for g in groups}
self._stop = threading.Event()
self._lock = threading.Lock()
- self._thread = threading.Thread(target = self._run, name = "unsloth-offload-pin", daemon = True)
+ self.after: list = []
+ self.late = False
+ self._thread = threading.Thread(target = self._entry, name = "unsloth-offload-pin", daemon = True)
self._started = False
for group in groups:
setattr(group, _BG_PIN_ATTR, self)
@@ -6762,6 +6765,34 @@ class _GroupPinner:
self._started = True
self._thread.start()
+ def _entry(self) -> None:
+ import time
+
+ began = time.perf_counter()
+ for dep in self.after:
+ dep.start()
+ while self.after and not self._stop.is_set():
+ if all(dep.join(0) for dep in self.after):
+ break
+ elapsed = time.perf_counter() - began
+ # Fast (not disk-bound) pins: pinning beside the others wins; overlapping the render only moves the cost.
+ rate = sum(max(dep.copied, dep.pinned) for dep in self.after) / max(elapsed, 1e-3)
+ if elapsed >= _LATE_PIN_PROBE_S and rate >= _LATE_PIN_PARALLEL_BPS:
+ self.late = False
+ if self.logger is not None:
+ self.logger.info(
+ "diffusion.memory: host weights pin at %.1f GiB/s; %s pins beside the others",
+ rate / 2**30,
+ self.label,
+ )
+ break
+ time.sleep(0.1)
+ if self._stop.is_set():
+ for event in self._done.values():
+ event.set()
+ return
+ self._run()
+
def wait(self, group: Any) -> None:
import threading
@@ -6805,6 +6836,15 @@ class _GroupPinner:
and not src.is_pinned()
]
+ def _submit(self, pool: Any, src: Any) -> Any:
+ future = pool.submit(_registered_pin, src, self.device)
+ future.add_done_callback(functools.partial(self._count_copy, int(src.nbytes)))
+ return future
+
+ def _count_copy(self, nbytes: int, future: Any) -> None:
+ if not future.cancelled():
+ self.copied += nbytes
+
@staticmethod
def _fast_result(future: Any, src: Any) -> Any:
try:
@@ -6887,7 +6927,7 @@ class _GroupPinner:
(
tensor,
src,
- pool.submit(_registered_pin, src, self.device),
+ self._submit(pool, src),
)
for tensor, src in self._unpinned(group)
]
@@ -6978,13 +7018,22 @@ def start_background_pins(pipe: Any) -> int:
return len(pinners)
-def finish_background_pins(pipe: Any, cancel: Any = None) -> float:
+def finish_background_pins(
+ pipe: Any,
+ cancel: Any = None,
+ include_late: bool = True,
+) -> float:
"""Blocks until deferred pinners finish, since a render overlapping one runs measurably slower."""
import time
start = time.perf_counter()
- for pinner in list(getattr(pipe, _PENDING_PINS_ATTR, None) or ()):
+ pinners = list(getattr(pipe, _PENDING_PINS_ATTR, None) or ())
+ for pinner in pinners:
pinner.start()
+ # Late pinners last: one may still turn early until the pinners before it finish.
+ for pinner in sorted(pinners, key = lambda p: bool(getattr(p, "late", False))):
+ if not include_late and getattr(pinner, "late", False):
+ continue
# Polled so a cancelled request leaves now; the pin itself keeps running for the next render.
while not pinner.join(0.25):
if cancel is not None and cancel.is_set():
@@ -6992,6 +7041,43 @@ def finish_background_pins(pipe: Any, cancel: Any = None) -> float:
return time.perf_counter() - start
+# "0": a two-expert denoiser's second expert pins beside the other weights, before the first render.
+LATE_EXPERT_PIN_ENV = "UNSLOTH_VIDEO_LATE_EXPERT_PIN"
+_LATE_PIN_PROBE_S = 0.2
+_LATE_PIN_PARALLEL_BPS = 3 << 30
+
+
+def defer_late_expert_pins(
+ pipe: Any,
+ late_modules: Any,
+ logger: Any = None,
+) -> int:
+ """Pin ``late_modules`` after the other pinners when disk-bound, so the render waits only for those; a late
+ group's onload still waits for its own pin. Wan2.2-A14B (A100): both experts at once made the first render wait
+ 47.8 s."""
+ if _env_off(LATE_EXPERT_PIN_ENV):
+ return 0
+ late_ids = {id(m) for m in late_modules or () if m is not None}
+ pending = list(getattr(pipe, _PENDING_PINS_ATTR, None) or ())
+ late = [p for p in pending if id(p.module) in late_ids]
+ early = [p for p in pending if id(p.module) not in late_ids]
+ if not late or not early:
+ return 0
+ for pinner in late:
+ pinner.after = list(early)
+ pinner.late = True
+ if logger is not None:
+ logger.info(
+ "diffusion.memory: %d late expert(s) (%.1f GiB) pin after the other %d pinner(s) (%.1f GiB); "
+ "the first render waits for those only",
+ len(late),
+ sum(p.outstanding_bytes() for p in late) / 2**30,
+ len(early),
+ sum(p.outstanding_bytes() for p in early) / 2**30,
+ )
+ return len(late)
+
+
def stop_background_pins(pipe: Any, timeout: Optional[float] = 30.0) -> None:
for pinner in list(getattr(pipe, _PENDING_PINS_ATTR, None) or ()):
try:
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 66cd00d610a..00e74810e4b 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -158,6 +158,7 @@ from .diffusion_memory import (
reclaim_offload_host_memory,
refine_memory_plan_for_components,
applied_plan,
+ defer_late_expert_pins,
finish_background_pins,
release_pinned_host_memory,
request_background_pins,
@@ -10203,6 +10204,8 @@ class VideoBackend:
effective_speed,
transformer_quant_engaged or "off",
)
+ if fam.is_moe:
+ defer_late_expert_pins(pipe, (getattr(pipe, "transformer_2", None),), logger)
start_background_pins(pipe)
if cast_writes is not None:
pipe._unsloth_cast_writes = cast_writes
@@ -11866,7 +11869,9 @@ class VideoBackend:
# First: on a fresh thread, un-indexed device calls resolve to the default card.
self._state_device_target(state)
# A render racing the post-load pinner ran every later render slower; the wait is usually over.
- waited = finish_background_pins(getattr(state, "pipe", None), cancel)
+ waited = finish_background_pins(
+ getattr(state, "pipe", None), cancel, include_late = False
+ )
if cancel.is_set():
raise RuntimeError(VIDEO_CANCELLED_MSG)
if waited >= 1.0:
diff --git a/studio/backend/tests/test_video_late_expert_pin.py b/studio/backend/tests/test_video_late_expert_pin.py
new file mode 100644
index 00000000000..1849d9ae58c
--- /dev/null
+++ b/studio/backend/tests/test_video_late_expert_pin.py
@@ -0,0 +1,198 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A two-expert denoiser's second expert pins after the other weights, and the render does not wait for it."""
+
+import threading
+import time
+
+import pytest
+
+import core.inference.diffusion_memory as mem
+
+
+class _Pipe:
+ pass
+
+
+class _Group:
+ cpu_param_dict: dict = {}
+
+
+class _Module:
+ def __init__(self, name):
+ self.name = name
+
+
+class _Recorder(mem._GroupPinner):
+ """Records when its pin ran instead of pinning; ``hold`` keeps it busy until set."""
+
+ def __init__(
+ self,
+ name,
+ log,
+ hold = None,
+ fast = False,
+ ):
+ super().__init__(_Module(name), [_Group()], None, None)
+ self.name = name
+ self.log = log
+ self.hold = hold
+ self.fast = fast
+
+ def _run(self):
+ self.log.append(("start", self.name))
+ if self.fast:
+ self.pinned = 64 << 30
+ if self.hold is not None:
+ self.hold.wait(10)
+ self.log.append(("end", self.name))
+ for event in self._done.values():
+ event.set()
+
+
+def _late(by, *names):
+ return tuple(by[n].module if n in by else _Module(n) for n in names)
+
+
+def _pipe(
+ log,
+ holds,
+ fast = False,
+):
+ pipe = _Pipe()
+ pinners = [
+ _Recorder(name, log, holds.get(name), fast)
+ for name in ("transformer", "transformer_2", "text_encoder")
+ ]
+ setattr(pipe, mem._PENDING_PINS_ATTR, pinners)
+ return pipe, {p.name: p for p in pinners}
+
+
+def test_late_expert_pins_after_the_others_and_the_render_skips_it(monkeypatch):
+ monkeypatch.delenv(mem.LATE_EXPERT_PIN_ENV, raising = False)
+ log: list = []
+ hold = threading.Event()
+ pipe, by = _pipe(log, {"text_encoder": hold})
+ assert mem.defer_late_expert_pins(pipe, _late(by, "transformer_2")) == 1
+ assert by["transformer_2"].late and not by["transformer"].late
+ mem.start_background_pins(pipe)
+ time.sleep(0.3)
+ assert ("start", "transformer_2") not in log
+ hold.set()
+ mem.finish_background_pins(pipe, include_late = False)
+ by["transformer_2"].join(10)
+ order = [m for kind, m in log if kind == "start"]
+ assert order[-1] == "transformer_2"
+ assert log.index(("start", "transformer_2")) > log.index(("end", "text_encoder"))
+ assert log.index(("start", "transformer_2")) > log.index(("end", "transformer"))
+
+
+def test_the_render_wait_returns_while_the_late_expert_is_still_pinning(monkeypatch):
+ monkeypatch.delenv(mem.LATE_EXPERT_PIN_ENV, raising = False)
+ log: list = []
+ hold = threading.Event()
+ pipe, by = _pipe(log, {"transformer_2": hold})
+ mem.defer_late_expert_pins(pipe, _late(by, "transformer_2"))
+ mem.start_background_pins(pipe)
+ began = time.perf_counter()
+ mem.finish_background_pins(pipe, include_late = False)
+ assert time.perf_counter() - began < 5
+ assert ("end", "transformer_2") not in log
+ hold.set()
+ mem.finish_background_pins(pipe)
+ assert ("end", "transformer_2") in log
+
+
+def test_an_early_onload_of_the_late_expert_waits_for_its_pin(monkeypatch):
+ monkeypatch.delenv(mem.LATE_EXPERT_PIN_ENV, raising = False)
+ log: list = []
+ pipe, by = _pipe(log, {})
+ mem.defer_late_expert_pins(pipe, _late(by, "transformer_2"))
+ late = by["transformer_2"]
+ late.wait(late.groups[0])
+ assert ("end", "transformer_2") in log
+ assert log.index(("start", "transformer_2")) > log.index(("end", "transformer"))
+
+
+def test_kill_switch_keeps_every_pinner_in_the_render_wait(monkeypatch):
+ monkeypatch.setenv(mem.LATE_EXPERT_PIN_ENV, "0")
+ log: list = []
+ pipe, by = _pipe(log, {})
+ assert mem.defer_late_expert_pins(pipe, _late(by, "transformer_2")) == 0
+ assert not any(p.late or p.after for p in by.values())
+ mem.start_background_pins(pipe)
+ mem.finish_background_pins(pipe, include_late = False)
+ assert all(("end", name) in log for name in by)
+
+
+ "late", [(), ("missing",), ("transformer", "transformer_2", "text_encoder")]
+)
+def test_nothing_is_deferred_without_both_a_late_and_an_early_pinner(monkeypatch, late):
+ monkeypatch.delenv(mem.LATE_EXPERT_PIN_ENV, raising = False)
+ pipe, by = _pipe([], {})
+ assert mem.defer_late_expert_pins(pipe, _late(by, *late)) == 0
+ assert not any(p.late for p in by.values())
+
+
+def test_a_stop_while_the_late_expert_waits_releases_its_groups(monkeypatch):
+ monkeypatch.delenv(mem.LATE_EXPERT_PIN_ENV, raising = False)
+ log: list = []
+ hold = threading.Event()
+ pipe, by = _pipe(log, {"transformer": hold})
+ mem.defer_late_expert_pins(pipe, _late(by, "transformer_2"))
+ mem.start_background_pins(pipe)
+ late = by["transformer_2"]
+ late.stop(5)
+ assert not late._thread.is_alive()
+ assert all(e.is_set() for e in late._done.values())
+ assert ("start", "transformer_2") not in log
+ hold.set()
+ mem.stop_background_pins(pipe)
+
+
+def test_fast_pins_run_the_late_expert_beside_the_others_and_the_render_waits_for_it(monkeypatch):
+ monkeypatch.delenv(mem.LATE_EXPERT_PIN_ENV, raising = False)
+ monkeypatch.setattr(mem, "_LATE_PIN_PROBE_S", 0.05)
+ log: list = []
+ hold = threading.Event()
+ pipe, by = _pipe(log, {"transformer": hold, "text_encoder": hold}, fast = True)
+ mem.defer_late_expert_pins(pipe, _late(by, "transformer_2"))
+ mem.start_background_pins(pipe)
+ deadline = time.time() + 5
+ while ("end", "transformer_2") not in log and time.time() < deadline:
+ time.sleep(0.02)
+ assert ("end", "transformer_2") in log and ("end", "transformer") not in log
+ assert not by["transformer_2"].late
+ hold.set()
+ mem.finish_background_pins(pipe, include_late = False)
+ assert not any(p._thread.is_alive() for p in by.values())
+
+
+def test_slow_pins_keep_the_late_expert_behind_the_others(monkeypatch):
+ monkeypatch.delenv(mem.LATE_EXPERT_PIN_ENV, raising = False)
+ monkeypatch.setattr(mem, "_LATE_PIN_PROBE_S", 0.05)
+ log: list = []
+ hold = threading.Event()
+ pipe, by = _pipe(log, {"transformer": hold})
+ mem.defer_late_expert_pins(pipe, _late(by, "transformer_2"))
+ mem.start_background_pins(pipe)
+ time.sleep(0.4)
+ assert ("start", "transformer_2") not in log and by["transformer_2"].late
+ hold.set()
+ mem.finish_background_pins(pipe)
+ assert log.index(("start", "transformer_2")) > log.index(("end", "transformer"))
+
+
+def test_copied_bytes_count_per_tensor_before_the_group_is_placed(monkeypatch):
+ from concurrent.futures import ThreadPoolExecutor
+
+ torch = pytest.importorskip("torch")
+ monkeypatch.setattr(mem, "_registered_pin", lambda src, device = None: src)
+ pinner = _Recorder("transformer", [])
+ with ThreadPoolExecutor(2) as pool:
+ futures = [pinner._submit(pool, torch.zeros(1024, dtype = torch.uint8)) for _ in range(3)]
+ for f in futures:
+ f.result()
+ assert pinner.copied == 3 * 1024 and pinner.pinned == 0