Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: stop the first Wan2.2-A14B render from waiting for the second expert's pin (#15906)
Daniel Han9258bfc73208
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex 5ecf1b9a38d..50d813a8a75 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -6747,11 +6747,14 @@ class _GroupPinner:self.device = deviceself.logger = loggerself.pinned = 0+ self.copied = 0self.waited_s = 0.0self._done = {id(g): threading.Event() for g in groups}self._stop = threading.Event()self._lock = threading.Lock()- self._thread = threading.Thread(target = self._run, name = "unsloth-offload-pin", daemon = True)+ self.after: list = []+ self.late = False+ self._thread = threading.Thread(target = self._entry, name = "unsloth-offload-pin", daemon = True)self._started = Falsefor group in groups:setattr(group, _BG_PIN_ATTR, self)@@ -6762,6 +6765,34 @@ class _GroupPinner:self._started = Trueself._thread.start()+ def _entry(self) -> None:+ import time++ began = time.perf_counter()+ for dep in self.after:+ dep.start()+ while self.after and not self._stop.is_set():+ if all(dep.join(0) for dep in self.after):+ break+ elapsed = time.perf_counter() - began+ # Fast (not disk-bound) pins: pinning beside the others wins; overlapping the render only moves the cost.+ rate = sum(max(dep.copied, dep.pinned) for dep in self.after) / max(elapsed, 1e-3)+ if elapsed >= _LATE_PIN_PROBE_S and rate >= _LATE_PIN_PARALLEL_BPS:+ self.late = False+ if self.logger is not None:+ self.logger.info(+ "diffusion.memory: host weights pin at %.1f GiB/s; %s pins beside the others",+ rate / 2**30,+ self.label,+ )+ break+ time.sleep(0.1)+ if self._stop.is_set():+ for event in self._done.values():+ event.set()+ return+ self._run()+def wait(self, group: Any) -> None:import threading@@ -6805,6 +6836,15 @@ class _GroupPinner:and not src.is_pinned()]+ def _submit(self, pool: Any, src: Any) -> Any:+ future = pool.submit(_registered_pin, src, self.device)+ future.add_done_callback(functools.partial(self._count_copy, int(src.nbytes)))+ return future++ def _count_copy(self, nbytes: int, future: Any) -> None:+ if not future.cancelled():+ self.copied += nbytes+@staticmethoddef _fast_result(future: Any, src: Any) -> Any:try:@@ -6887,7 +6927,7 @@ class _GroupPinner:(tensor,src,- pool.submit(_registered_pin, src, self.device),+ self._submit(pool, src),)for tensor, src in self._unpinned(group)]@@ -6978,13 +7018,22 @@ def start_background_pins(pipe: Any) -> int:return len(pinners)-def finish_background_pins(pipe: Any, cancel: Any = None) -> float:+def finish_background_pins(+ pipe: Any,+ cancel: Any = None,+ include_late: bool = True,+) -> float:"""Blocks until deferred pinners finish, since a render overlapping one runs measurably slower."""import timestart = time.perf_counter()- for pinner in list(getattr(pipe, _PENDING_PINS_ATTR, None) or ()):+ pinners = list(getattr(pipe, _PENDING_PINS_ATTR, None) or ())+ for pinner in pinners:pinner.start()+ # Late pinners last: one may still turn early until the pinners before it finish.+ for pinner in sorted(pinners, key = lambda p: bool(getattr(p, "late", False))):+ if not include_late and getattr(pinner, "late", False):+ continue# Polled so a cancelled request leaves now; the pin itself keeps running for the next render.while not pinner.join(0.25):if cancel is not None and cancel.is_set():@@ -6992,6 +7041,43 @@ def finish_background_pins(pipe: Any, cancel: Any = None) -> float:return time.perf_counter() - start+# "0": a two-expert denoiser's second expert pins beside the other weights, before the first render.+LATE_EXPERT_PIN_ENV = "UNSLOTH_VIDEO_LATE_EXPERT_PIN"+_LATE_PIN_PROBE_S = 0.2+_LATE_PIN_PARALLEL_BPS = 3 << 30+++def defer_late_expert_pins(+ pipe: Any,+ late_modules: Any,+ logger: Any = None,+) -> int:+ """Pin ``late_modules`` after the other pinners when disk-bound, so the render waits only for those; a late+ group's onload still waits for its own pin. Wan2.2-A14B (A100): both experts at once made the first render wait+ 47.8 s."""+ if _env_off(LATE_EXPERT_PIN_ENV):+ return 0+ late_ids = {id(m) for m in late_modules or () if m is not None}+ pending = list(getattr(pipe, _PENDING_PINS_ATTR, None) or ())+ late = [p for p in pending if id(p.module) in late_ids]+ early = [p for p in pending if id(p.module) not in late_ids]+ if not late or not early:+ return 0+ for pinner in late:+ pinner.after = list(early)+ pinner.late = True+ if logger is not None:+ logger.info(+ "diffusion.memory: %d late expert(s) (%.1f GiB) pin after the other %d pinner(s) (%.1f GiB); "+ "the first render waits for those only",+ len(late),+ sum(p.outstanding_bytes() for p in late) / 2**30,+ len(early),+ sum(p.outstanding_bytes() for p in early) / 2**30,+ )+ return len(late)++def stop_background_pins(pipe: Any, timeout: Optional[float] = 30.0) -> None:for pinner in list(getattr(pipe, _PENDING_PINS_ATTR, None) or ()):try:diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 66cd00d610a..00e74810e4b 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -158,6 +158,7 @@ from .diffusion_memory import (reclaim_offload_host_memory,refine_memory_plan_for_components,applied_plan,+ defer_late_expert_pins,finish_background_pins,release_pinned_host_memory,request_background_pins,@@ -10203,6 +10204,8 @@ class VideoBackend:effective_speed,transformer_quant_engaged or "off",)+ if fam.is_moe:+ defer_late_expert_pins(pipe, (getattr(pipe, "transformer_2", None),), logger)start_background_pins(pipe)if cast_writes is not None:pipe._unsloth_cast_writes = cast_writes@@ -11866,7 +11869,9 @@ class VideoBackend:# First: on a fresh thread, un-indexed device calls resolve to the default card.self._state_device_target(state)# A render racing the post-load pinner ran every later render slower; the wait is usually over.- waited = finish_background_pins(getattr(state, "pipe", None), cancel)+ waited = finish_background_pins(+ getattr(state, "pipe", None), cancel, include_late = False+ )if cancel.is_set():raise RuntimeError(VIDEO_CANCELLED_MSG)if waited >= 1.0:diff --git a/studio/backend/tests/test_video_late_expert_pin.py b/studio/backend/tests/test_video_late_expert_pin.pynew file mode 100644index 00000000000..1849d9ae58c--- /dev/null+++ b/studio/backend/tests/test_video_late_expert_pin.py@@ -0,0 +1,198 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A two-expert denoiser's second expert pins after the other weights, and the render does not wait for it."""++import threading+import time++import pytest++import core.inference.diffusion_memory as mem+++class _Pipe:+ pass+++class _Group:+ cpu_param_dict: dict = {}+++class _Module:+ def __init__(self, name):+ self.name = name+++class _Recorder(mem._GroupPinner):+ """Records when its pin ran instead of pinning; ``hold`` keeps it busy until set."""++ def __init__(+ self,+ name,+ log,+ hold = None,+ fast = False,+ ):+ super().__init__(_Module(name), [_Group()], None, None)+ self.name = name+ self.log = log+ self.hold = hold+ self.fast = fast++ def _run(self):+ self.log.append(("start", self.name))+ if self.fast:+ self.pinned = 64 << 30+ if self.hold is not None:+ self.hold.wait(10)+ self.log.append(("end", self.name))+ for event in self._done.values():+ event.set()+++def _late(by, *names):+ return tuple(by[n].module if n in by else _Module(n) for n in names)+++def _pipe(+ log,+ holds,+ fast = False,+):+ pipe = _Pipe()+ pinners = [+ _Recorder(name, log, holds.get(name), fast)+ for name in ("transformer", "transformer_2", "text_encoder")+ ]+ setattr(pipe, mem._PENDING_PINS_ATTR, pinners)+ return pipe, {p.name: p for p in pinners}+++def test_late_expert_pins_after_the_others_and_the_render_skips_it(monkeypatch):+ monkeypatch.delenv(mem.LATE_EXPERT_PIN_ENV, raising = False)+ log: list = []+ hold = threading.Event()+ pipe, by = _pipe(log, {"text_encoder": hold})+ assert mem.defer_late_expert_pins(pipe, _late(by, "transformer_2")) == 1+ assert by["transformer_2"].late and not by["transformer"].late+ mem.start_background_pins(pipe)+ time.sleep(0.3)+ assert ("start", "transformer_2") not in log+ hold.set()+ mem.finish_background_pins(pipe, include_late = False)+ by["transformer_2"].join(10)+ order = [m for kind, m in log if kind == "start"]+ assert order[-1] == "transformer_2"+ assert log.index(("start", "transformer_2")) > log.index(("end", "text_encoder"))+ assert log.index(("start", "transformer_2")) > log.index(("end", "transformer"))+++def test_the_render_wait_returns_while_the_late_expert_is_still_pinning(monkeypatch):+ monkeypatch.delenv(mem.LATE_EXPERT_PIN_ENV, raising = False)+ log: list = []+ hold = threading.Event()+ pipe, by = _pipe(log, {"transformer_2": hold})+ mem.defer_late_expert_pins(pipe, _late(by, "transformer_2"))+ mem.start_background_pins(pipe)+ began = time.perf_counter()+ mem.finish_background_pins(pipe, include_late = False)+ assert time.perf_counter() - began < 5+ assert ("end", "transformer_2") not in log+ hold.set()+ mem.finish_background_pins(pipe)+ assert ("end", "transformer_2") in log+++def test_an_early_onload_of_the_late_expert_waits_for_its_pin(monkeypatch):+ monkeypatch.delenv(mem.LATE_EXPERT_PIN_ENV, raising = False)+ log: list = []+ pipe, by = _pipe(log, {})+ mem.defer_late_expert_pins(pipe, _late(by, "transformer_2"))+ late = by["transformer_2"]+ late.wait(late.groups[0])+ assert ("end", "transformer_2") in log+ assert log.index(("start", "transformer_2")) > log.index(("end", "transformer"))+++def test_kill_switch_keeps_every_pinner_in_the_render_wait(monkeypatch):+ monkeypatch.setenv(mem.LATE_EXPERT_PIN_ENV, "0")+ log: list = []+ pipe, by = _pipe(log, {})+ assert mem.defer_late_expert_pins(pipe, _late(by, "transformer_2")) == 0+ assert not any(p.late or p.after for p in by.values())+ mem.start_background_pins(pipe)+ mem.finish_background_pins(pipe, include_late = False)+ assert all(("end", name) in log for name in by)+++ "late", [(), ("missing",), ("transformer", "transformer_2", "text_encoder")]+)+def test_nothing_is_deferred_without_both_a_late_and_an_early_pinner(monkeypatch, late):+ monkeypatch.delenv(mem.LATE_EXPERT_PIN_ENV, raising = False)+ pipe, by = _pipe([], {})+ assert mem.defer_late_expert_pins(pipe, _late(by, *late)) == 0+ assert not any(p.late for p in by.values())+++def test_a_stop_while_the_late_expert_waits_releases_its_groups(monkeypatch):+ monkeypatch.delenv(mem.LATE_EXPERT_PIN_ENV, raising = False)+ log: list = []+ hold = threading.Event()+ pipe, by = _pipe(log, {"transformer": hold})+ mem.defer_late_expert_pins(pipe, _late(by, "transformer_2"))+ mem.start_background_pins(pipe)+ late = by["transformer_2"]+ late.stop(5)+ assert not late._thread.is_alive()+ assert all(e.is_set() for e in late._done.values())+ assert ("start", "transformer_2") not in log+ hold.set()+ mem.stop_background_pins(pipe)+++def test_fast_pins_run_the_late_expert_beside_the_others_and_the_render_waits_for_it(monkeypatch):+ monkeypatch.delenv(mem.LATE_EXPERT_PIN_ENV, raising = False)+ monkeypatch.setattr(mem, "_LATE_PIN_PROBE_S", 0.05)+ log: list = []+ hold = threading.Event()+ pipe, by = _pipe(log, {"transformer": hold, "text_encoder": hold}, fast = True)+ mem.defer_late_expert_pins(pipe, _late(by, "transformer_2"))+ mem.start_background_pins(pipe)+ deadline = time.time() + 5+ while ("end", "transformer_2") not in log and time.time() < deadline:+ time.sleep(0.02)+ assert ("end", "transformer_2") in log and ("end", "transformer") not in log+ assert not by["transformer_2"].late+ hold.set()+ mem.finish_background_pins(pipe, include_late = False)+ assert not any(p._thread.is_alive() for p in by.values())+++def test_slow_pins_keep_the_late_expert_behind_the_others(monkeypatch):+ monkeypatch.delenv(mem.LATE_EXPERT_PIN_ENV, raising = False)+ monkeypatch.setattr(mem, "_LATE_PIN_PROBE_S", 0.05)+ log: list = []+ hold = threading.Event()+ pipe, by = _pipe(log, {"transformer": hold})+ mem.defer_late_expert_pins(pipe, _late(by, "transformer_2"))+ mem.start_background_pins(pipe)+ time.sleep(0.4)+ assert ("start", "transformer_2") not in log and by["transformer_2"].late+ hold.set()+ mem.finish_background_pins(pipe)+ assert log.index(("start", "transformer_2")) > log.index(("end", "transformer"))+++def test_copied_bytes_count_per_tensor_before_the_group_is_placed(monkeypatch):+ from concurrent.futures import ThreadPoolExecutor++ torch = pytest.importorskip("torch")+ monkeypatch.setattr(mem, "_registered_pin", lambda src, device = None: src)+ pinner = _Recorder("transformer", [])+ with ThreadPoolExecutor(2) as pool:+ futures = [pinner._submit(pool, torch.zeros(1024, dtype = torch.uint8)) for _ in range(3)]+ for f in futures:+ f.result()+ assert pinner.copied == 3 * 1024 and pinner.pinned == 0